diff --git a/.charts.yml b/.charts.yml
index a4ff95a..cecb75d 100644
--- a/.charts.yml
+++ b/.charts.yml
@@ -1,16 +1,26 @@
+.common:
+  openstack_helm_repository: &openstack_helm_repository
+    url: https://tarballs.openstack.org/openstack-helm
+  openstack_helm_infra_repository: &openstack_helm_infra_repository
+    url: https://tarballs.openstack.org/openstack-helm-infra
+  openstack_helm_dependencies: &openstack_helm_dependencies
+    - name: helm-toolkit
+      repository: https://tarballs.openstack.org/openstack-helm-infra
+      version: 0.2.64
+
 charts:
   - name: barbican
     version: 0.3.6
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
   - name: ceph-csi-rbd
     version: 3.5.1
     repository:
       url: https://ceph.github.io/csi-charts
   - name: ceph-provisioners
     version: 0.1.8
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm-infra
+    repository: *openstack_helm_infra_repository
+    dependencies: *openstack_helm_dependencies
   - name: godaddy-webhook
     version: 0.3.0
     repository:
@@ -21,12 +31,8 @@
       url: https://luisico.github.io/cert-manager-webhook-infoblox-wapi
   - name: cinder
     version: 0.3.15
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -37,48 +43,32 @@
       url: https://coredns.github.io/helm
   - name: designate
     version: 0.2.9
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
           - 899932
   - name: glance
     version: 0.4.15
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
           - 899864
   - name: heat
     version: 0.3.7
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
           - 899931
   - name: horizon
     version: 0.3.15
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
   - name: ingress-nginx
     version: 4.0.17
     repository:
@@ -89,12 +79,8 @@
       url: https://charts.bitnami.com/bitnami
   - name: keystone
     version: 0.3.5
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -105,12 +91,8 @@
       url: https://prometheus-community.github.io/helm-charts
   - name: libvirt
     version: 0.1.27
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm-infra
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.56
+    repository: *openstack_helm_infra_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -121,24 +103,16 @@
       url: https://grafana.github.io/helm-charts
   - name: magnum
     version: 0.2.9
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
           - 899926
   - name: manila
     version: 0.1.7
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -146,16 +120,12 @@
           - 899923
   - name: memcached
     version: 0.1.12
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm-infra
+    repository: *openstack_helm_infra_repository
+    dependencies: *openstack_helm_dependencies
   - name: neutron
     version: 0.3.29
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -167,12 +137,8 @@
       url: https://kubernetes-sigs.github.io/node-feature-discovery/charts
   - name: nova
     version: 0.3.27
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -180,32 +146,20 @@
           - 904250
   - name: octavia
     version: 0.2.9
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
           - 899918
   - name: openvswitch
     version: 0.1.19
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm-infra
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.56
+    repository: *openstack_helm_infra_repository
+    dependencies: *openstack_helm_dependencies
   - name: ovn
     version: 0.1.4
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm-infra
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.56
+    repository: *openstack_helm_infra_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -213,12 +167,8 @@
           - 914807
   - name: placement
     version: 0.3.9
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
@@ -245,24 +195,16 @@
       url: https://charts.rook.io/release
   - name: senlin
     version: 0.2.9
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
     patches:
       gerrit:
         review.opendev.org:
           - 899913
   - name: tempest
     version: 0.2.8
-    repository:
-      url: https://tarballs.openstack.org/openstack-helm
-    dependencies:
-      - name: helm-toolkit
-        repository: https://tarballs.openstack.org/openstack-helm-infra
-        version: 0.2.55
+    repository: *openstack_helm_repository
+    dependencies: *openstack_helm_dependencies
   - name: vector
     version: 0.32.0
     repository:
diff --git a/charts/barbican/charts/helm-toolkit/Chart.yaml b/charts/barbican/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/barbican/charts/helm-toolkit/Chart.yaml
+++ b/charts/barbican/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/barbican/charts/helm-toolkit/requirements.lock b/charts/barbican/charts/helm-toolkit/requirements.lock
new file mode 100644
index 0000000..808bd94
--- /dev/null
+++ b/charts/barbican/charts/helm-toolkit/requirements.lock
@@ -0,0 +1,3 @@
+dependencies: []
+digest: sha256:643d5437104296e21d906ecb15b2c96ad278f20cfc4af53b12bb6069bd853726
+generated: '0001-01-01T00:00:00Z'
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/barbican/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/barbican/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/barbican/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/barbican/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/barbican/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/barbican/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/barbican/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/barbican/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/barbican/requirements.lock b/charts/barbican/requirements.lock
index b9146d7..43aa382 100644
--- a/charts/barbican/requirements.lock
+++ b/charts/barbican/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
-  repository: file://../../openstack-helm-infra/helm-toolkit
-  version: 0.2.55
-digest: sha256:f25556fa4f0b285a96cbf853e72555c04e65772daf0ffa6b518321e5e249f5ca
-generated: "2023-11-15T23:08:24.10384684Z"
+  repository: https://tarballs.openstack.org/openstack-helm-infra
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
+generated: '0001-01-01T00:00:00Z'
diff --git a/charts/barbican/requirements.yaml b/charts/barbican/requirements.yaml
index 4124d01..ddafbfc 100644
--- a/charts/barbican/requirements.yaml
+++ b/charts/barbican/requirements.yaml
@@ -1,16 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 dependencies:
-  - name: helm-toolkit
-    repository: file://../../openstack-helm-infra/helm-toolkit
-    version: ">= 0.1.0"
+- name: helm-toolkit
+  repository: https://tarballs.openstack.org/openstack-helm-infra
+  version: 0.2.64
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml b/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml
index f762292..e827e99 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml
+++ b/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.17
+version: 0.2.64
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/requirements.lock b/charts/ceph-provisioners/charts/helm-toolkit/requirements.lock
index df843bb..808bd94 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/requirements.lock
+++ b/charts/ceph-provisioners/charts/helm-toolkit/requirements.lock
@@ -1,3 +1,3 @@
 dependencies: []
 digest: sha256:643d5437104296e21d906ecb15b2c96ad278f20cfc4af53b12bb6069bd853726
-generated: "2021-07-27T15:44:21.585311483Z"
+generated: '0001-01-01T00:00:00Z'
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl
new file mode 100644
index 0000000..cc4d4de
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl
@@ -0,0 +1,38 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Resolves the namespace scoped hostname for an endpoint
+values: |
+  endpoints:
+    oslo_db:
+      hosts:
+        default: mariadb
+      host_fqdn_override:
+        default: null
+usage: |
+  {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" }}
+return: |
+  default
+*/}}
+
+{{- define "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" -}}
+{{- $type := index . 0 -}}
+{{- $endpoint := index . 1 -}}
+{{- $context := index . 2 -}}
+{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }}
+{{- $namespace := $endpointMap.namespace | default $context.Release.Namespace }}
+{{- printf "%s" $namespace -}}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_certificates.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_certificates.tpl
index 241e8b1..8be771e 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_certificates.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_certificates.tpl
@@ -30,7 +30,8 @@
                   organization:
                     - ACME
                   commonName: keystone-api.openstack.svc.cluster.local
-                  keySize: 2048
+                  privateKey:
+                    size: 2048
                   usages:
                     - server auth
                     - client auth
@@ -55,55 +56,8 @@
         duration: 2160h
         issuerRef:
           name: ca-issuer
-        keySize: 2048
-        organization:
-        - ACME
-        secretName: keystone-tls-api
-        usages:
-        - server auth
-        - client auth
-
-  - values: |
-      cert_manager_version: v0.15.0
-      endpoints:
-        dashboard:
-          host_fqdn_override:
-            default:
-              host: null
-              tls:
-                secretName: keystone-tls-api
-                issuerRef:
-                  name: ca-issuer
-                  duration: 2160h
-                  organization:
-                    - ACME
-                  commonName: keystone-api.openstack.svc.cluster.local
-                  keySize: 2048
-                  usages:
-                    - server auth
-                    - client auth
-                  dnsNames:
-                    - cluster.local
-                  issuerRef:
-                    name: ca-issuer
-    usage: |
-      {{- $opts := dict "envAll" . "service" "dashboard" "type" "internal" -}}
-      {{ $opts | include "helm-toolkit.manifests.certificates" }}
-    return: |
-      ---
-      apiVersion: cert-manager.io/v1alpha3
-      kind: Certificate
-      metadata:
-        name: keystone-tls-api
-        namespace: NAMESPACE
-      spec:
-        commonName: keystone-api.openstack.svc.cluster.local
-        dnsNames:
-        - cluster.local
-        duration: 2160h
-        issuerRef:
-          name: ca-issuer
-        keySize: 2048
+        privateKey:
+          size: 2048
         organization:
         - ACME
         secretName: keystone-tls-api
@@ -125,11 +79,13 @@
 {{- $dnsNames := list $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) -}}
 {{- $_ := $dnsNames | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "dnsNames" -}}
 {{- end -}}
-{{/* Default keySize to 4096. This can be overridden. */}}
-{{- if not (hasKey $slice "keySize") -}}
-{{- $_ := ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "keySize" -}}
+{{/* Default privateKey size to 4096. This can be overridden. */}}
+{{- if not (hasKey $slice "privateKey") -}}
+{{- $_ := dict "size" ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "privateKey" -}}
+{{- else if empty (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "privateKey" "size") -}}
+{{- $_ := ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "privateKey") "size" -}}
 {{- end -}}
-{{/* Default keySize to 3 months. Note the min is 720h. This can be overridden. */}}
+{{/* Default duration to 3 months. Note the min is 720h. This can be overridden. */}}
 {{- if not (hasKey $slice "duration") -}}
 {{- $_ := printf "%s" "2190h" | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "duration" -}}
 {{- end -}}
@@ -141,16 +97,8 @@
 {{- if not (hasKey $slice "usages") -}}
 {{- $_ := (list "server auth" "client auth") | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "usages" -}}
 {{- end -}}
-{{- $cert_manager_version := "v1.0.0" -}}
-{{- if $envAll.Values.cert_manager_version -}}
-{{- $cert_manager_version = $envAll.Values.cert_manager_version -}}
-{{- end -}}
 ---
-{{- if semverCompare "< v1.0.0"  $cert_manager_version }}
-apiVersion: cert-manager.io/v1alpha3
-{{- else }}
 apiVersion: cert-manager.io/v1
-{{- end }}
 kind: Certificate
 metadata:
   name: {{ index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "secretName" }}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 2d62a17..cacb4b8 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,50 +59,59 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         rules:
           - host: barbican
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default.svc.cluster.local
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: barbican-namespace-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public
             hosts:
@@ -112,19 +121,22 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: barbican-cluster-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx-cluster"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx-cluster"
         tls:
           - secretName: barbican-tls-public
             hosts:
@@ -134,9 +146,12 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
   - values: |
       network:
         api:
@@ -179,18 +194,18 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public
             hosts:
@@ -202,23 +217,32 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default.svc.cluster.local
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
   - values: |
       cert_issuer_type: issuer
       network:
@@ -270,20 +294,20 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           cert-manager.io/issuer: ca-issuer
           certmanager.k8s.io/issuer: ca-issuer
           nginx.ingress.kubernetes.io/backend-protocol: https
           nginx.ingress.kubernetes.io/secure-backends: "true"
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public-certmanager
             hosts:
@@ -295,23 +319,32 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default.svc.cluster.local
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
 
   - values: |
       network:
@@ -363,20 +396,20 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           cert-manager.io/cluster-issuer: ca-issuer
           certmanager.k8s.io/cluster-issuer: ca-issuer
           nginx.ingress.kubernetes.io/backend-protocol: https
           nginx.ingress.kubernetes.io/secure-backends: "true"
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public-certmanager
             hosts:
@@ -388,23 +421,32 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
           - host: barbican.default.svc.cluster.local
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: barbican-api
-                    servicePort: b-api
+                    service:
+                      name: barbican-api
+                      port:
+                        name: b-api
   # Sample usage for multiple DNS names associated with the same public
   # endpoint and certificate
   - values: |
@@ -437,51 +479,60 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: grafana
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         rules:
           - host: grafana
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
           - host: grafana.default
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
           - host: grafana.default.svc.cluster.local
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: grafana-namespace-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: grafana-tls-public
             hosts:
@@ -492,26 +543,32 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
           - host: grafana-alt.openstackhelm.example
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
       ---
-      apiVersion: networking.k8s.io/v1beta1
+      apiVersion: networking.k8s.io/v1
       kind: Ingress
       metadata:
         name: grafana-cluster-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx-cluster"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx-cluster"
         tls:
           - secretName: grafana-tls-public
             hosts:
@@ -522,16 +579,22 @@
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
           - host: grafana-alt.openstackhelm.example
             http:
               paths:
                 - path: /
+                  pathType: Prefix
                   backend:
-                    serviceName: grafana-dashboard
-                    servicePort: dashboard
+                    service:
+                      name: grafana-dashboard
+                      port:
+                        name: dashboard
 
 */}}
 
@@ -539,13 +602,21 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
+        pathType: {{ $pathType }}
         backend:
-          serviceName: {{ $backendName }}
-          servicePort: {{ $backendPort }}
+          service:
+            name: {{ $backendName }}
+            port:
+{{- if or (kindIs "int" $backendPort) (regexMatch "^[0-9]{1,5}$" $backendPort) }}
+              number: {{ $backendPort | int }}
+{{- else }}
+              name: {{ $backendPort | quote }}
+{{- end }}
 {{- end }}
 
 {{- define "helm-toolkit.manifests.ingress" -}}
@@ -554,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -564,12 +636,11 @@
 {{- $certIssuerType = $envAll.Values.cert_issuer_type }}
 {{- end }}
 ---
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ $ingressName }}
   annotations:
-    kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }}
 {{- if $certIssuer }}
     cert-manager.io/{{ $certIssuerType }}: {{ $certIssuer }}
     certmanager.k8s.io/{{ $certIssuerType }}: {{ $certIssuer }}
@@ -580,6 +651,7 @@
 {{- end }}
 {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }}
 spec:
+  ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }}
 {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "hosts" }}
 {{- if $certIssuer }}
 {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
@@ -611,21 +683,23 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
-{{- range $key2, $ingressController := tuple "namespace" "cluster" }}
+{{- $ingressConf := $envAll.Values.network -}}
+{{- $ingressClasses := ternary (tuple "namespace") (tuple "namespace" "cluster") (and (hasKey $ingressConf "use_external_ingress_controller") $ingressConf.use_external_ingress_controller) }}
+{{- range $key2, $ingressController := $ingressClasses }}
 {{- $vHosts := list $hostNameFull }}
 ---
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ printf "%s-%s-%s" $ingressName $ingressController "fqdn" }}
   annotations:
-    kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }}
 {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }}
 spec:
+  ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }}
 {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "host_fqdn_override" }}
 {{- if hasKey $host $endpoint }}
 {{- $endpointHost := index $host $endpoint }}
@@ -634,7 +708,6 @@
 {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }}
 {{- $vHosts = append $vHosts $v }}
 {{- end }}
-{{- if and ( not ( empty $endpointHost.tls.key ) ) ( not ( empty $endpointHost.tls.crt ) ) }}
 {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }}
   tls:
@@ -646,10 +719,9 @@
 {{- end }}
 {{- end }}
 {{- end }}
-{{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 6bd0898..6b77004 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -23,6 +23,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $podVolMounts := index . "podVolMounts" | default false -}}
 {{- $podVols := index . "podVols" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
@@ -44,7 +45,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "bootstrap" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -65,8 +72,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "bootstrap" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index cfd64ff..2b7ff2c 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -28,6 +28,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}}
 {{- $dbToDrop := index . "dbToDrop" | default ( dict "adminSecret" $envAll.Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" ) -}}
@@ -45,9 +46,15 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "db-drop" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "db-drop" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -66,8 +73,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "db_drop" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "db_drop" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 4463397..b8a1dce 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -28,6 +28,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}}
 {{- $dbToInit := index . "dbToInit" | default ( dict "adminSecret" $envAll.Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" ) -}}
@@ -45,7 +46,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "db-init" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -66,8 +73,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "db_init" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "db_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 979211d..4696c88 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -23,6 +23,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}}
 {{- $podVolMounts := index . "podVolMounts" | default false -}}
@@ -42,7 +43,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "db-sync" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "db-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -63,8 +70,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "db_sync" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "db_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 6df37b6..d69c9e6 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -24,6 +24,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $secretBin := index . "secretBin" -}}
 {{- $tlsSecret := index . "tlsSecret" | default "" -}}
@@ -45,7 +46,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "ks-endpoints" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "ks-endpoints" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -66,8 +73,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: {{ $restartPolicy }}
+      {{ tuple $envAll "ks_endpoints" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "ks_endpoints" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index ca9f6c3..9604c63 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -24,6 +24,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $secretBin := index . "secretBin" -}}
 {{- $tlsSecret := index . "tlsSecret" | default "" -}}
@@ -45,7 +46,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "ks-service" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "ks-service" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -66,8 +73,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: {{ $restartPolicy }}
+      {{ tuple $envAll "ks_service" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "ks_service" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 42f2370..58dcdc5 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -17,12 +17,35 @@
 # {- $ksUserJob := dict "envAll" . "serviceName" "senlin" }
 # { $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }
 
+{{/*
+  # To enable PodSecuritycontext (PodSecurityContext/v1) define the below in values.yaml:
+  # example:
+  #  values: |
+  #    pod:
+  #      security_context:
+  #        ks_user:
+  #          pod:
+  #            runAsUser: 65534
+  # To enable Container SecurityContext(SecurityContext/v1) for ks-user container define the values:
+  # example:
+  #   values: |
+  #     pod:
+  #       security_context:
+  #         ks_user:
+  #           container:
+  #             ks-user:
+  #               runAsUser: 65534
+  #               readOnlyRootFilesystem: true
+  #               allowPrivilegeEscalation: false
+*/}}
+
 {{- define "helm-toolkit.manifests.job_ks_user" -}}
 {{- $envAll := index . "envAll" -}}
 {{- $serviceName := index . "serviceName" -}}
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $serviceUser := index . "serviceUser" | default $serviceName -}}
 {{- $secretBin := index . "secretBin" -}}
@@ -45,7 +68,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceUserPretty "ks-user" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -65,9 +94,14 @@
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
     spec:
       serviceAccountName: {{ $serviceAccountName | quote }}
+{{ dict "envAll" $envAll "application" "ks_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       restartPolicy: {{ $restartPolicy }}
+      {{ tuple $envAll "ks_user" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "ks_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
@@ -75,6 +109,7 @@
           image: {{ $envAll.Values.images.tags.ks_user }}
           imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "ks_user" "container" "ks_user" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /bin/bash
             - -c
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 5574032..2cfadaf 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -18,6 +18,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $serviceUser := index . "serviceUser" | default $serviceName -}}
 {{- $secretBin := index . "secretBin" -}}
@@ -35,7 +36,13 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceUserPretty "rabbit-init" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "rabbit-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -56,8 +63,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName | quote }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "rabbit_init" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "rabbit_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
@@ -94,7 +105,7 @@
           - name: RABBITMQ_AUXILIARY_CONFIGURATION
             value: {{ toJson $envAll.Values.conf.rabbitmq | quote }}
 {{- end }}
-{{- if $envAll.Values.manifests.certificates }}
+{{- if and $envAll.Values.manifests.certificates (ne $tlsSecret "") }}
           - name: RABBITMQ_X509
             value: "REQUIRE X509"
           - name: USER_CERT_PATH
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index bea6876..b5fdc09 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -23,6 +23,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $configMapCeph := index . "configMapCeph" | default (printf "ceph-etc" ) -}}
 {{- $secretBin := index . "secretBin" -}}
@@ -41,8 +42,14 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "s3-bucket" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "s3-bucket" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -61,8 +68,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName | quote }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "s3_bucket" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "s3_bucket" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 36af63f..77d1a71 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -23,6 +23,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
 {{- $configMapCeph := index . "configMapCeph" | default (printf "ceph-etc" ) -}}
 {{- $secretBin := index . "secretBin" -}}
@@ -38,9 +39,15 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "s3-user" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "s3-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
@@ -59,8 +66,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName | quote }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "s3_user" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "s3_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
         - name: ceph-keyring-placement
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl
index 2e67006..0906df4 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl
@@ -23,6 +23,7 @@
 {{- $jobAnnotations := index . "jobAnnotations" -}}
 {{- $jobLabels := index . "jobLabels" -}}
 {{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
+{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}}
 {{- $podVolMounts := index . "podVolMounts" | default false -}}
 {{- $podVols := index . "podVols" | default false -}}
 {{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
@@ -38,6 +39,11 @@
 kind: Job
 metadata:
   name: {{ printf "%s-%s" $serviceNamePretty "image-repo-sync" | quote }}
+  labels:
+{{ tuple $envAll $serviceName "image-repo-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- if $jobLabels }}
+{{ toYaml $jobLabels | indent 4 }}
+{{- end }}
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
 {{- if $jobAnnotations }}
@@ -58,8 +64,12 @@
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
+      {{ tuple $envAll "image_repo_sync" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }}
       nodeSelector:
 {{ toYaml $nodeSelector | indent 8 }}
+{{- if $tolerationsEnabled }}
+{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{- end}}
       initContainers:
 {{ tuple $envAll "image_repo_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
       containers:
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
new file mode 100644
index 0000000..7ad505b
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -0,0 +1,78 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Creates a manifest for a authenticating a registry with a secret
+examples:
+  - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
+      secrets:
+        oci_image_registry:
+          {{ $serviceName }}: {{ $keyName }}
+      endpoints:
+        oci_image_registry:
+          name: oci-image-registry
+          auth:
+            enabled: true
+             {{ $serviceName }}:
+                name: {{ $userName }}
+                password: {{ $password }}
+  usage: |
+    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
+  return: |
+    ---
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
+    type: kubernetes.io/dockerconfigjson
+    data:
+      dockerconfigjson: {{ $dockerAuth }}
+*/}}
+
+{{- define "helm-toolkit.manifests.secret_registry" }}
+{{- $envAll := index . "envAll" }}
+{{- $registryUser := index . "registryUser" }}
+{{- $secretName := index $envAll.Values.secrets.oci_image_registry $registryUser }}
+{{- $registryHost := tuple "oci_image_registry" "internal" $envAll | include "helm-toolkit.endpoints.endpoint_host_lookup" }}
+{{/*
+We only use "host:port" when port is non-null, else just use "host"
+*/}}
+{{- $registryPort := "" }}
+{{- $port := $envAll.Values.endpoints.oci_image_registry.port.registry.default }}
+{{- if $port }}
+{{- $port = tuple "oci_image_registry" "internal" "registry" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $registryPort = printf ":%s" $port }}
+{{- end }}
+{{- $imageCredentials := index $envAll.Values.endpoints.oci_image_registry.auth $registryUser }}
+{{- $dockerAuthToken := printf "%s:%s" $imageCredentials.username $imageCredentials.password | b64enc }}
+{{- $dockerAuth := printf "{\"auths\": {\"%s%s\": {\"auth\": \"%s\"}}}" $registryHost $registryPort $dockerAuthToken | b64enc }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: {{ $dockerAuth }}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl
index 4294d40..6027b95 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl
@@ -133,8 +133,10 @@
 # Create DB User
 try:
     root_engine.execute(
-        "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\' IDENTIFIED BY \'{2}\' {3}".format(
-            database, user, password, mysql_x509))
+        "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format(
+            user, password, mysql_x509))
+    root_engine.execute(
+        "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))
     logger.info("Created user {0} for {1}".format(user, database))
 except:
     logger.critical("Could not create user {0} for {1}".format(user, database))
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl
index 93cea25..4d7dfaa 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl
@@ -63,4 +63,7 @@
 
 #give permissions to user
 pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $USER_DB_NAME to $USER_DB_USER;"
+
+#revoke all privileges from PUBLIC role
+pgsql_superuser_cmd "REVOKE ALL ON DATABASE $USER_DB_NAME FROM PUBLIC;"
 {{- end }}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl
index 87872d6..3739f95 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl
@@ -77,6 +77,11 @@
   password="${RABBITMQ_PASSWORD}" \
   tags="user"
 
+echo "Deleting Guest User"
+rabbitmqadmin_cli \
+  delete user \
+  name="guest" || true
+
 if [ "${RABBITMQ_VHOST}" != "/" ]
 then
   echo "Managing: vHost: ${RABBITMQ_VHOST}"
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 7c62bc4..695cb2e 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -40,11 +40,21 @@
 #           export OS_PROJECT_DOMAIN_NAME  Keystone domain the user belongs to
 #           export OS_IDENTITY_API_VERSION Keystone API version to use
 #
-#       The following variables are optional:
-#         export RGW_TIMEOUT           Number of seconds to wait for the
-#                                      connection to the RGW to be available
-#                                      when sending a backup to the RGW. Default
-#                                      is 1800 (30 minutes).
+#           export REMOTE_BACKUP_RETRIES   Number of retries to send backup to remote
+#                                          in case of any temporary failures.
+#           export MIN_DELAY_SEND_REMOTE   Minimum seconds to delay before sending backup
+#                                          to remote to stagger backups being sent to RGW
+#           export MAX_DELAY_SEND_REMOTE   Maximum seconds to delay before sending backup
+#                                          to remote to stagger backups being sent to RGW.
+#                                          A random number between min and max delay is generated
+#                                          to set the delay.
+#
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
 #
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
@@ -63,6 +73,14 @@
 #       framework will automatically tar/zip the files in that directory and
 #       name the tarball appropriately according to the proper conventions.
 #
+#   verify_databases_backup_archives [scope]
+#       returns: 0 if no errors; 1 if any errors occurred
+#
+#       This function is expected to verify the database backup archives. If this function
+#        completes successfully (returns 0), the
+#       framework will automatically starts remote backup upload.
+#
+#
 # The functions in this file will take care of:
 #   1) Calling "dump_databases_to_directory" and then compressing the files,
 #      naming the tarball properly, and then storing it locally at the specified
@@ -73,21 +91,32 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
-set -x
 
 log_backup_error_exit() {
   MSG=$1
-  ERRCODE=$2
+  ERRCODE=${2:-0}
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
+log_verify_backup_exit() {
+  MSG=$1
+  ERRCODE=${2:-0}
+  log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
+  rm -f $ERR_LOG_FILE
+  # rm -rf $TMP_DIR
+  exit 0
+}
+
+
 log() {
   #Log message to a file or stdout
   #TODO: This can be convert into mail alert of alert send to a monitoring system
@@ -107,6 +136,13 @@
   fi
 }
 
+# Generate a random number between MIN_DELAY_SEND_REMOTE and
+# MAX_DELAY_SEND_REMOTE
+random_number() {
+  diff=$((${MAX_DELAY_SEND_REMOTE} - ${MIN_DELAY_SEND_REMOTE} + 1))
+  echo $(($(( ${RANDOM} % ${diff} )) + ${MIN_DELAY_SEND_REMOTE} ))
+}
+
 #Get the day delta since the archive file backup
 seconds_difference() {
   ARCHIVE_DATE=$( date --date="$1" +%s )
@@ -135,9 +171,17 @@
     if [[ $? -ne 0 ]]; then
       # Find the swift URL from the keystone endpoint list
       SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+      if [[ $? -ne 0 ]]; then
+        log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+        return 2
+      fi
 
       # Get a token from keystone
       TOKEN=$(openstack token issue -f value -c id)
+      if [[ $? -ne 0 ]]; then
+        log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+        return 2
+      fi
 
       # Create the container
       RES_FILE=$(mktemp -p /tmp)
@@ -146,28 +190,28 @@
            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
 
       if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
-        log ERROR "${DB_NAME}_backup" "Error creating container ${CONTAINER_NAME}"
+        log WARN "${DB_NAME}_backup" "Unable to create container ${CONTAINER_NAME}"
         cat $RES_FILE
         rm -f $RES_FILE
-        return 1
+        return 2
       fi
       rm -f $RES_FILE
 
       swift stat $CONTAINER_NAME
       if [[ $? -ne 0 ]]; then
-        log ERROR "${DB_NAME}_backup" "Error retrieving container ${CONTAINER_NAME} details after creation."
-        return 1
+        log WARN "${DB_NAME}_backup" "Unable to retrieve container ${CONTAINER_NAME} details after creation."
+        return 2
       fi
     fi
   else
-    echo $RESULT | grep "HTTP 401"
+    echo $RESULT | grep -E "HTTP 401|HTTP 403"
     if [[ $? -eq 0 ]]; then
       log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
       return 1
     else
-      echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable"
+      echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
       if [[ $? -eq 0 ]]; then
-        log ERROR "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+        log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
         # In this case, keystone or the site/node may be temporarily down.
         # Return slightly different error code so the calling code can retry
         return 2
@@ -178,14 +222,163 @@
     fi
   fi
 
+  # load balance delay
+  DELAY=$((1 + ${RANDOM} % 30))
+  echo "Sleeping for ${DELAY} seconds to spread the load in time..."
+  sleep ${DELAY}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
-  openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE || log ERROR "${DB_NAME}_backup" "Cannot create container object ${FILE}!"
+  openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
+  if [[ $? -ne 0 ]]; then
+    log WARN "${DB_NAME}_backup" "Cannot create container object ${FILE}!"
+    return 2
+  fi
+
   openstack object show $CONTAINER_NAME $FILE
   if [[ $? -ne 0 ]]; then
-    log ERROR "${DB_NAME}_backup" "Error retrieving container object $FILE after creation."
-    return 1
+    log WARN "${DB_NAME}_backup" "Unable to retrieve container object $FILE after creation."
+    return 2
   fi
 
+  # Remote backup verification
+  MD5_REMOTE=$(openstack object show $CONTAINER_NAME $FILE -f json | jq -r ".etag")
+  MD5_LOCAL=$(cat ${FILEPATH}/${FILE} | md5sum | awk '{print $1}')
+  log INFO "${DB_NAME}_backup" "Obtained MD5 hash for the file $FILE in container $CONTAINER_NAME."
+  log INFO "${DB_NAME}_backup" "Local MD5 hash is ${MD5_LOCAL}."
+  log INFO "${DB_NAME}_backup" "Remote MD5 hash is ${MD5_REMOTE}."
+  if [[ "${MD5_LOCAL}" == "${MD5_REMOTE}" ]]; then
+      log INFO "${DB_NAME}_backup" "The local backup & remote backup MD5 hash values are matching for file $FILE in container $CONTAINER_NAME."
+  else
+      log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
+      return 2
+  fi
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
+
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
 }
@@ -198,93 +391,175 @@
   FILEPATH=$1
   FILE=$2
 
-  # If the RGW_TIMEOUT has already been set, use that value, otherwise give it
-  # a default value.
-  if [[ -z $RGW_TIMEOUT ]]; then
-    RGW_TIMEOUT=1800
-  fi
-
-  ERROR_SEEN=false
-  DONE=false
-  TIMEOUT_EXP=$(( $(date +%s) + $RGW_TIMEOUT ))
-  while [[ $DONE == "false" ]]; do
+  count=1
+  while [[ ${count} -le ${REMOTE_BACKUP_RETRIES} ]]; do
     # Store the new archive to the remote backup storage facility.
     send_to_remote_server $FILEPATH $FILE
+    SEND_RESULT="$?"
 
     # Check if successful
-    if [[ $? -eq 0 ]]; then
+    if [[ $SEND_RESULT -eq 0 ]]; then
       log INFO "${DB_NAME}_backup" "Backup file ${FILE} successfully sent to RGW."
-      DONE=true
-    elif [[ $? -eq 2 ]]; then
-      # Temporary failure occurred. We need to retry if we have not timed out
-      log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to RGW due to connection issue."
-      DELTA=$(( TIMEOUT_EXP - $(date +%s) ))
-      if [[ $DELTA -lt 0 ]]; then
-        DONE=true
-        log ERROR "${DB_NAME}_backup" "Timed out waiting for RGW to become available."
-        ERROR_SEEN=true
-      else
-        log INFO "${DB_NAME}_backup" "Sleeping 30 seconds waiting for RGW to become available..."
-        sleep 30
-        log INFO "${DB_NAME}_backup" "Retrying..."
+      return 0
+    elif [[ $SEND_RESULT -eq 2 ]]; then
+      if [[ ${count} -ge ${REMOTE_BACKUP_RETRIES} ]]; then
+        log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW in " \
+        "${REMOTE_BACKUP_RETRIES} retries. Errors encountered. Exiting."
+        break
       fi
+      # Temporary failure occurred. We need to retry
+      log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to RGW due to connection issue."
+      sleep_time=$(random_number)
+      log INFO "${DB_NAME}_backup" "Sleeping ${sleep_time} seconds waiting for RGW to become available..."
+      sleep ${sleep_time}
+      log INFO "${DB_NAME}_backup" "Retrying..."
     else
-      log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW."
-      ERROR_SEEN=true
-      DONE=true
+      log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW. Errors encountered. Exiting."
+      break
     fi
+
+    # Increment the counter
+    count=$((count+1))
   done
 
-  if [[ $ERROR_SEEN == "true" ]]; then
-    log ERROR "${DB_NAME}_backup" "Errors encountered. Exiting."
-    return 1
-  fi
-  return 0
+  return 1
+}
+
+
+function get_archive_date(){
+# get_archive_date function returns correct archive date
+# for different formats of archives' names
+# the old one: <database name>.<namespace>.<table name | all>.<date-time>.tar.gz
+# the new one: <database name>.<namespace>.<table name | all>.<backup mode>.<date-time>.tar.gz
+  local A_FILE="$1"
+  awk -F. '{print $(NF-2)}' <<< ${A_FILE} | tr -d "Z"
+}
+
+# This function takes a list of archives' names as an input
+# and creates a hash table where keys are number of seconds
+# between current date and archive date (see seconds_difference),
+# and values are space separated archives' names
+#
+# +------------+---------------------------------------------------------------------------------------------------------+
+# | 1265342678 | "tmp/mysql.backup.auto.2022-02-14T10:13:13Z.tar.gz"                                                     |
+# +------------+---------------------------------------------------------------------------------------------------------+
+# | 2346254257 | "tmp/mysql.backup.auto.2022-02-11T10:13:13Z.tar.gz tmp/mysql.backup.manual.2022-02-11T10:13:13Z.tar.gz" |
+# +------------+---------------------------------------------------------------------------------------------------------+
+# <...>
+# +------------+---------------------------------------------------------------------------------------------------------+
+# | 6253434567 | "tmp/mysql.backup.manual.2022-02-01T10:13:13Z.tar.gz"                                                   |
+# +------------+---------------------------------------------------------------------------------------------------------+
+# We will use the explained above data stracture to cover rare, but still
+# possible case, when we have several backups of the same date. E.g.
+# one manual, and one automatic.
+
+declare -A fileTable
+create_hash_table() {
+unset fileTable
+fileList=$@
+  for ARCHIVE_FILE in ${fileList}; do
+    # Creating index, we will round given ARCHIVE_DATE to the midnight (00:00:00)
+    # to take in account a possibility, that we can have more than one scheduled
+    # backup per day.
+    ARCHIVE_DATE=$(get_archive_date ${ARCHIVE_FILE})
+    ARCHIVE_DATE=$(date --date=${ARCHIVE_DATE} +%D)
+    log INFO "${DB_NAME}_backup" "Archive date to build index: ${ARCHIVE_DATE}"
+    INDEX=$(seconds_difference ${ARCHIVE_DATE})
+    if [[ -z fileTable[${INDEX}] ]]; then
+      fileTable[${INDEX}]=${ARCHIVE_FILE}
+    else
+      fileTable[${INDEX}]="${fileTable[${INDEX}]} ${ARCHIVE_FILE}"
+    fi
+    echo "INDEX: ${INDEX} VALUE:  ${fileTable[${INDEX}]}"
+ done
+}
+
+function get_backup_prefix() {
+# Create list of all possible prefixes in a format:
+# <db_name>.<namespace> to cover a possible situation
+# when different backups of different databases and/or
+# namespaces share the same local or remote storage.
+  ALL_FILES=($@)
+  PREFIXES=()
+  for fname in ${ALL_FILES[@]}; do
+    prefix=$(basename ${fname} | cut -d'.' -f1,2 )
+    for ((i=0; i<${#PREFIXES[@]}; i++)) do
+      if [[ ${PREFIXES[${i}]} == ${prefix} ]]; then
+        prefix=""
+        break
+      fi
+    done
+    if [[ ! -z ${prefix} ]]; then
+        PREFIXES+=(${prefix})
+    fi
+  done
 }
 
 remove_old_local_archives() {
-  log INFO "${DB_NAME}_backup" "Deleting backups older than ${LOCAL_DAYS_TO_KEEP} days"
+  SECONDS_TO_KEEP=$(( $((${LOCAL_DAYS_TO_KEEP}))*86400))
+  log INFO "${DB_NAME}_backup" "Deleting backups older than ${LOCAL_DAYS_TO_KEEP} days (${SECONDS_TO_KEEP} seconds)"
   if [[ -d $ARCHIVE_DIR ]]; then
-    for ARCHIVE_FILE in $(ls -1 $ARCHIVE_DIR/*.gz); do
-      ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
-      if [[ "$(seconds_difference $ARCHIVE_DATE)" -gt "$(($LOCAL_DAYS_TO_KEEP*86400))" ]]; then
-        log INFO "${DB_NAME}_backup" "Deleting file $ARCHIVE_FILE."
-        rm -rf $ARCHIVE_FILE
-        if [[ $? -ne 0 ]]; then
-          # Log error but don't exit so we can finish the script
-          # because at this point we haven't sent backup to RGW yet
-          log ERROR "${DB_NAME}_backup" "Cannot remove ${ARCHIVE_FILE}"
-        fi
+    count=0
+    # We iterate over the hash table, checking the delta in seconds (hash keys),
+    # and minimum number of backups we must have in place. List of keys has to be sorted.
+    for INDEX in $(tr " " "\n" <<< ${!fileTable[@]} | sort -n -); do
+      ARCHIVE_FILE=${fileTable[${INDEX}]}
+      if [[ ${INDEX} -lt ${SECONDS_TO_KEEP} || ${count} -lt ${LOCAL_DAYS_TO_KEEP} ]]; then
+        ((count++))
+        log INFO "${DB_NAME}_backup" "Keeping file(s) ${ARCHIVE_FILE}."
       else
-        log INFO "${DB_NAME}_backup" "Keeping file ${ARCHIVE_FILE}."
+        log INFO "${DB_NAME}_backup" "Deleting file(s) ${ARCHIVE_FILE}."
+          rm -f ${ARCHIVE_FILE}
+          if [[ $? -ne 0 ]]; then
+            # Log error but don't exit so we can finish the script
+            # because at this point we haven't sent backup to RGW yet
+            log ERROR "${DB_NAME}_backup" "Failed to cleanup local backup. Cannot remove some of ${ARCHIVE_FILE}"
+          fi
       fi
     done
+  else
+    log WARN "${DB_NAME}_backup" "The local backup directory ${$ARCHIVE_DIR} does not exist."
   fi
 }
 
-remove_old_remote_archives() {
-  log INFO "${DB_NAME}_backup" "Deleting backups older than ${REMOTE_DAYS_TO_KEEP} days"
+prepare_list_of_remote_backups() {
   BACKUP_FILES=$(mktemp -p /tmp)
   DB_BACKUP_FILES=$(mktemp -p /tmp)
-
   openstack object list $CONTAINER_NAME > $BACKUP_FILES
   if [[ $? -ne 0 ]]; then
-    log_backup_error_exit "Could not obtain a list of current backup files in the RGW" 1
+    log_backup_error_exit \
+      "Failed to cleanup remote backup. Could not obtain a list of current backup files in the RGW"
   fi
-
   # Filter out other types of backup files
   cat $BACKUP_FILES | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $DB_BACKUP_FILES
+}
 
-  for ARCHIVE_FILE in $(cat $DB_BACKUP_FILES); do
-    ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4)
-    if [[ "$(seconds_difference ${ARCHIVE_DATE})" -gt "$((${REMOTE_DAYS_TO_KEEP}*86400))" ]]; then
-      log INFO "${DB_NAME}_backup" "Deleting file ${ARCHIVE_FILE} from the RGW"
-      openstack object delete $CONTAINER_NAME $ARCHIVE_FILE || log_backup_error_exit "Cannot delete container object ${ARCHIVE_FILE}!" 1
+# The logic implemented with this function is absolutely similar
+# to the function remove_old_local_archives (see above)
+remove_old_remote_archives() {
+  count=0
+  SECONDS_TO_KEEP=$((${REMOTE_DAYS_TO_KEEP}*86400))
+  log INFO "${DB_NAME}_backup" "Deleting backups older than ${REMOTE_DAYS_TO_KEEP} days (${SECONDS_TO_KEEP} seconds)"
+  for INDEX in $(tr " " "\n" <<< ${!fileTable[@]} | sort -n -); do
+    ARCHIVE_FILE=${fileTable[${INDEX}]}
+    if [[ ${INDEX} -lt ${SECONDS_TO_KEEP} || ${count} -lt ${REMOTE_DAYS_TO_KEEP} ]]; then
+      ((count++))
+      log INFO "${DB_NAME}_backup" "Keeping remote backup(s) ${ARCHIVE_FILE}."
+    else
+      log INFO "${DB_NAME}_backup" "Deleting remote backup(s) ${ARCHIVE_FILE} from the RGW"
+      openstack object delete ${CONTAINER_NAME} ${ARCHIVE_FILE} ||  log WARN "${DB_NAME}_backup" \
+        "Failed to cleanup remote backup. Cannot delete container object ${ARCHIVE_FILE}"
     fi
   done
 
   # Cleanup now that we're done.
-  rm -f $BACKUP_FILES $DB_BACKUP_FILES
+  for fd in ${BACKUP_FILES} ${DB_BACKUP_FILES}; do
+    if [[ -f ${fd} ]]; then
+      rm -f ${fd}
+    else
+      log WARN "${DB_NAME}_backup" "Can not delete a temporary file ${fd}"
+    fi
+  done
 }
 
 # Main function to backup the databases. Calling functions need to supply:
@@ -297,11 +572,14 @@
   SCOPE=${1:-"all"}
 
   # Create necessary directories if they do not exist.
-  mkdir -p $ARCHIVE_DIR || log_backup_error_exit "Cannot create directory ${ARCHIVE_DIR}!"
-  export TMP_DIR=$(mktemp -d) || log_backup_error_exit "Cannot create temp directory!"
+  mkdir -p $ARCHIVE_DIR || log_backup_error_exit \
+    "Backup of the ${DB_NAME} database failed. Cannot create directory ${ARCHIVE_DIR}!"
+  export TMP_DIR=$(mktemp -d) || log_backup_error_exit \
+    "Backup of the ${DB_NAME} database failed. Cannot create temp directory!"
 
   # Create temporary log file
-  export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit "Cannot create log file!"
+  export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit \
+    "Backup of the ${DB_NAME} database failed. Cannot create log file!"
 
   # It is expected that this function will dump the database files to the $TMP_DIR
   dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE
@@ -321,12 +599,14 @@
     TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${BACK_UP_MODE}.${NOW}.tar.gz"
   fi
 
-  cd $TMP_DIR || log_backup_error_exit "Cannot change to directory $TMP_DIR"
+  cd $TMP_DIR || log_backup_error_exit \
+    "Backup of the ${DB_NAME} database failed. Cannot change to directory $TMP_DIR"
 
   #Archive the current database files
   tar zcvf $ARCHIVE_DIR/$TARBALL_FILE *
   if [[ $? -ne 0 ]]; then
-    log_backup_error_exit "Backup tarball could not be created."
+    log_backup_error_exit \
+      "Backup ${DB_NAME} to local file system failed. Backup tarball could not be created."
   fi
 
   # Get the size of the file
@@ -336,30 +616,55 @@
 
   cd $ARCHIVE_DIR
 
+  #Only delete the old archive after a successful archive
+  export LOCAL_DAYS_TO_KEEP=$(echo $LOCAL_DAYS_TO_KEEP | sed 's/"//g')
+  if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then
+    get_backup_prefix $(ls -1 ${ARCHIVE_DIR}/*.gz)
+    for ((i=0; i<${#PREFIXES[@]}; i++)); do
+      echo "Working with prefix: ${PREFIXES[i]}"
+      create_hash_table $(ls -1 ${ARCHIVE_DIR}/${PREFIXES[i]}*.gz)
+      remove_old_local_archives
+    done
+  fi
+
+  # Local backup verification process
+
+  # It is expected that this function will verify the database backup files
+  if verify_databases_backup_archives ${SCOPE}; then
+    log INFO "${DB_NAME}_backup_verify" "Databases backup verified successfully. Uploading verified backups to remote location..."
+  else
+    # If successful, there should be at least one file in the TMP_DIR
+    if [[ $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
+      cat $ERR_LOG_FILE
+    fi
+    log_verify_backup_exit "Verify of the ${DB_NAME} database backup failed and needs attention."
+    exit 1
+  fi
+
   # Remove the temporary directory and files as they are no longer needed.
   rm -rf $TMP_DIR
   rm -f $ERR_LOG_FILE
 
-  #Only delete the old archive after a successful archive
-  export LOCAL_DAYS_TO_KEEP=$(echo $LOCAL_DAYS_TO_KEEP | sed 's/"//g')
-  if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then
-    remove_old_local_archives
-  fi
-
+  # Remote backup
   REMOTE_BACKUP=$(echo $REMOTE_BACKUP_ENABLED | sed 's/"//g')
   if $REMOTE_BACKUP; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export REMOTE_BACKUP_RETRIES=$(echo $REMOTE_BACKUP_RETRIES | sed 's/"//g')
+    export MIN_DELAY_SEND_REMOTE=$(echo $MIN_DELAY_SEND_REMOTE | sed 's/"//g')
+    export MAX_DELAY_SEND_REMOTE=$(echo $MAX_DELAY_SEND_REMOTE | sed 's/"//g')
+    export REMOTE_DAYS_TO_KEEP=$(echo $REMOTE_DAYS_TO_KEEP | sed 's/"//g')
+
     store_backup_remotely $ARCHIVE_DIR $TARBALL_FILE
     if [[ $? -ne 0 ]]; then
       # This error should print first, then print the summary as the last
       # thing that the user sees in the output.
       log ERROR "${DB_NAME}_backup" "Backup ${TARBALL_FILE} could not be sent to remote RGW."
-      set +x
       echo "=================================================================="
       echo "Local backup successful, but could not send to remote RGW."
       echo "Backup archive name: $TARBALL_FILE"
       echo "Backup archive size: $ARCHIVE_SIZE"
       echo "=================================================================="
-      set -x
       # Because the local backup was successful, exit with 0 so the pod will not
       # continue to restart and fill the disk with more backups. The ERRORs are
       # logged and alerting system should catch those errors and flag the operator.
@@ -367,31 +672,30 @@
     fi
 
     #Only delete the old archive after a successful archive
-    export REMOTE_DAYS_TO_KEEP=$(echo $REMOTE_DAYS_TO_KEEP | sed 's/"//g')
     if [[ "$REMOTE_DAYS_TO_KEEP" -gt 0 ]]; then
-      remove_old_remote_archives
+      prepare_list_of_remote_backups
+      get_backup_prefix $(cat $DB_BACKUP_FILES)
+      for ((i=0; i<${#PREFIXES[@]}; i++)); do
+        echo "Working with prefix: ${PREFIXES[i]}"
+        create_hash_table $(cat ${DB_BACKUP_FILES} | grep ${PREFIXES[i]})
+        remove_old_remote_archives
+      done
     fi
 
-    # Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
-    set +x
     echo "=================================================================="
     echo "Local backup and backup to remote RGW successful!"
     echo "Backup archive name: $TARBALL_FILE"
     echo "Backup archive size: $ARCHIVE_SIZE"
     echo "=================================================================="
-    set -x
   else
     # Remote backup is not enabled. This is ok; at least we have a local backup.
     log INFO "${DB_NAME}_backup" "Skipping remote backup, as it is not enabled."
 
-    # Turn off trace just for a clearer printout of backup status - for manual backups, mainly.
-    set +x
     echo "=================================================================="
     echo "Local backup successful!"
     echo "Backup archive name: $TARBALL_FILE"
     echo "Backup archive size: $ARCHIVE_SIZE"
     echo "=================================================================="
-    set -x
   fi
 }
-{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl
index c2de3aa..093dd2c 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl
@@ -269,7 +269,7 @@
       echo "=============================================="
       for archive in $archives
       do
-        echo $archive | cut -d '/' -f 8
+        echo $archive | cut -d '/' -f8-
       done
       clean_and_exit 0 ""
     else
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl
index 0324e68..48b53fa 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl
@@ -17,12 +17,20 @@
   Renders a set of standardised labels
 values: |
   release_group: null
+  pod:
+    labels:
+      default:
+        label1.example.com: value
+      bar:
+        label2.example.com: bar
 usage: |
   {{ tuple . "foo" "bar" | include "helm-toolkit.snippets.kubernetes_metadata_labels" }}
 return: |
   release_group: RELEASE-NAME
   application: foo
   component: bar
+  label1.example.com: value
+  label2.example.com: bar
 */}}
 
 {{- define "helm-toolkit.snippets.kubernetes_metadata_labels" -}}
@@ -32,4 +40,12 @@
 release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }}
 application: {{ $application }}
 component: {{ $component }}
+{{- if ($envAll.Values.pod).labels }}
+{{- if hasKey $envAll.Values.pod.labels $component }}
+{{ index $envAll.Values.pod "labels" $component | toYaml }}
+{{- end -}}
+{{- if hasKey $envAll.Values.pod.labels "default" }}
+{{ $envAll.Values.pod.labels.default | toYaml }}
+{{- end -}}
+{{- end -}}
 {{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_image_pull_secret.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_image_pull_secret.tpl
new file mode 100644
index 0000000..74173dc
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_image_pull_secret.tpl
@@ -0,0 +1,45 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Renders image pull secrets for a pod
+values: |
+  pod:
+    image_pull_secrets:
+      default:
+        - name: some-pull-secret
+      bar:
+        - name: another-pull-secret
+usage: |
+  {{ tuple . "bar" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" }}
+return: |
+  imagePullSecrets:
+    - name: some-pull-secret
+    - name: another-pull-secret
+*/}}
+
+{{- define "helm-toolkit.snippets.kubernetes_image_pull_secrets" -}}
+{{- $envAll := index . 0 -}}
+{{- $application := index . 1 -}}
+{{- if ($envAll.Values.pod).image_pull_secrets }}
+imagePullSecrets:
+{{- if hasKey $envAll.Values.pod.image_pull_secrets $application }}
+{{ index $envAll.Values.pod "image_pull_secrets" $application | toYaml | indent 2 }}
+{{- end -}}
+{{- if hasKey $envAll.Values.pod.image_pull_secrets "default" }}
+{{ $envAll.Values.pod.image_pull_secrets.default | toYaml | indent 2 }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl
index baa7073..90a7a65 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl
@@ -62,5 +62,8 @@
       - services
       - endpoints
       {{- end -}}
+      {{ if eq $v "secrets" }}
+      - secrets
+      {{- end -}}
       {{- end -}}
 {{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl
index a8f1c49..bc2045e 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl
@@ -42,6 +42,12 @@
 metadata:
   name: {{ $saName }}
   namespace: {{ $saNamespace }}
+{{- if $envAll.Values.manifests.secret_registry }}
+{{- if $envAll.Values.endpoints.oci_image_registry.auth.enabled }}
+imagePullSecrets:
+  - name: {{ index $envAll.Values.secrets.oci_image_registry $envAll.Chart.Name }}
+{{- end -}}
+{{- end -}}
 {{- range $k, $v := $deps -}}
 {{- if eq $k "services" }}
 {{- range $serv := $v }}
@@ -57,6 +63,8 @@
 {{- $_ := set $allNamespace $saNamespace  (printf "%s%s" "daemonsets," ((index $allNamespace $saNamespace) | default "")) }}
 {{- else if and (eq $k "pod") $v }}
 {{- $_ := set $allNamespace $saNamespace  (printf "%s%s" "pods," ((index $allNamespace $saNamespace) | default "")) }}
+{{- else if and (eq $k "secret") $v }}
+{{- $_ := set $allNamespace $saNamespace  (printf "%s%s" "secrets," ((index $allNamespace $saNamespace) | default "")) }}
 {{- end -}}
 {{- end -}}
 {{- $_ := unset $allNamespace $randomKey }}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl
new file mode 100644
index 0000000..fc74c6f
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl
@@ -0,0 +1,68 @@
+{{- define "helm-toolkit.snippets.mon_host_from_k8s_ep" -}}
+{{/*
+
+Inserts a bash function definition mon_host_from_k8s_ep() which can be used
+to construct a mon_hosts value from the given namespaced endpoint.
+
+Usage (e.g. in _script.sh.tpl):
+    #!/bin/bash
+
+    : "${NS:=ceph}"
+    : "${EP:=ceph-mon-discovery}"
+
+    {{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+
+    MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP")
+
+    if [ -z "$MON_HOST" ]; then
+        # deal with failure
+    else
+        sed -i -e "s/^mon_host = /mon_host = $MON_HOST/" /etc/ceph/ceph.conf
+    fi
+*/}}
+{{`
+# Construct a mon_hosts value from the given namespaced endpoint
+# IP x.x.x.x with port p named "mon-msgr2" will appear as [v2:x.x.x.x/p/0]
+# IP x.x.x.x with port q named "mon" will appear as [v1:x.x.x.x/q/0]
+# IP x.x.x.x with ports p and q will appear as [v2:x.x.x.x/p/0,v1:x.x.x.x/q/0]
+# The entries for all IPs will be joined with commas
+mon_host_from_k8s_ep() {
+  local ns=$1
+  local ep=$2
+
+  if [ -z "$ns" ] || [ -z "$ep" ]; then
+    return 1
+  fi
+
+  # We don't want shell expansion for the go-template expression
+  # shellcheck disable=SC2016
+  kubectl get endpoints -n "$ns" "$ep" -o go-template='
+    {{- $sep := "" }}
+    {{- range $_,$s := .subsets }}
+      {{- $v2port := 0 }}
+      {{- $v1port := 0 }}
+      {{- range $_,$port := index $s "ports" }}
+        {{- if (eq $port.name "mon-msgr2") }}
+          {{- $v2port = $port.port }}
+        {{- else if (eq $port.name "mon") }}
+          {{- $v1port = $port.port }}
+        {{- end }}
+      {{- end }}
+      {{- range $_,$address := index $s "addresses" }}
+        {{- $v2endpoint := printf "v2:%s:%d/0" $address.ip $v2port }}
+        {{- $v1endpoint := printf "v1:%s:%d/0" $address.ip $v1port }}
+        {{- if (and $v2port $v1port) }}
+          {{- printf "%s[%s,%s]" $sep $v2endpoint $v1endpoint }}
+          {{- $sep = "," }}
+        {{- else if $v2port }}
+          {{- printf "%s[%s]" $sep $v2endpoint }}
+          {{- $sep = "," }}
+        {{- else if $v1port }}
+          {{- printf "%s[%s]" $sep $v1endpoint }}
+          {{- $sep = "," }}
+        {{- end }}
+      {{- end }}
+    {{- end }}'
+}
+`}}
+{{- end -}}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl
index b99c00d..4a88dd8 100644
--- a/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl
+++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl
@@ -27,10 +27,12 @@
 {{- else if kindIs "slice" $dependencyMixinParam }}
 {{- $_ := set $envAll.Values "__deps" ( index $envAll.Values.dependencies.static $dependencyKey ) }}
 {{- range $k, $v := $dependencyMixinParam -}}
+{{- if ( index $envAll.Values.dependencies.dynamic.targeted $v ) }}
 {{- $_ := include "helm-toolkit.utils.merge" (tuple $envAll.Values.pod_dependency $envAll.Values.__deps ( index $envAll.Values.dependencies.dynamic.targeted $v $dependencyKey ) ) -}}
 {{- $_ := set $envAll.Values "__deps" $envAll.Values.pod_dependency -}}
 {{- end }}
 {{- end }}
+{{- end }}
 {{- else -}}
 {{- $_ := set $envAll.Values "pod_dependency" ( index $envAll.Values.dependencies.static $dependencyKey ) -}}
 {{- end -}}
diff --git a/charts/ceph-provisioners/requirements.lock b/charts/ceph-provisioners/requirements.lock
index 3b09b9a..43aa382 100644
--- a/charts/ceph-provisioners/requirements.lock
+++ b/charts/ceph-provisioners/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
-  repository: https://tarballs.opendev.org/openstack/openstack-helm-infra
-  version: '>= 0.1.0'
-digest: sha256:9a14200f65f9e9d7f811f6b763242eea2a0ff5f36199412abc2c58f273b95899
-generated: "2021-07-28T15:09:21.187908283Z"
+  repository: https://tarballs.openstack.org/openstack-helm-infra
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
+generated: '0001-01-01T00:00:00Z'
diff --git a/charts/ceph-provisioners/requirements.yaml b/charts/ceph-provisioners/requirements.yaml
index 4333ba9..ddafbfc 100644
--- a/charts/ceph-provisioners/requirements.yaml
+++ b/charts/ceph-provisioners/requirements.yaml
@@ -1,18 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
 dependencies:
-  - name: helm-toolkit
-    repository: https://tarballs.opendev.org/openstack/openstack-helm-infra
-    version: ">= 0.1.0"
-...
+- name: helm-toolkit
+  repository: https://tarballs.openstack.org/openstack-helm-infra
+  version: 0.2.64
diff --git a/charts/cinder/charts/helm-toolkit/Chart.yaml b/charts/cinder/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/cinder/charts/helm-toolkit/Chart.yaml
+++ b/charts/cinder/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/cinder/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/cinder/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/cinder/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/cinder/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/cinder/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/cinder/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/cinder/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/cinder/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/cinder/requirements.lock b/charts/cinder/requirements.lock
index e346dde..43aa382 100644
--- a/charts/cinder/requirements.lock
+++ b/charts/cinder/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/cinder/requirements.yaml b/charts/cinder/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/cinder/requirements.yaml
+++ b/charts/cinder/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/designate/charts/helm-toolkit/Chart.yaml b/charts/designate/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/designate/charts/helm-toolkit/Chart.yaml
+++ b/charts/designate/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/designate/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/designate/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/designate/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/designate/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/designate/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/designate/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/designate/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/designate/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/designate/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/designate/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/designate/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/designate/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/designate/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/designate/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/designate/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/designate/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/designate/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/designate/requirements.lock b/charts/designate/requirements.lock
index e346dde..43aa382 100644
--- a/charts/designate/requirements.lock
+++ b/charts/designate/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/designate/requirements.yaml b/charts/designate/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/designate/requirements.yaml
+++ b/charts/designate/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/glance/charts/helm-toolkit/Chart.yaml b/charts/glance/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/glance/charts/helm-toolkit/Chart.yaml
+++ b/charts/glance/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/glance/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/glance/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/glance/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/glance/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/glance/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/glance/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/glance/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/glance/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/glance/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/glance/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/glance/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/glance/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/glance/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/glance/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/glance/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/glance/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/glance/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/glance/requirements.lock b/charts/glance/requirements.lock
index e346dde..43aa382 100644
--- a/charts/glance/requirements.lock
+++ b/charts/glance/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/glance/requirements.yaml b/charts/glance/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/glance/requirements.yaml
+++ b/charts/glance/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/heat/charts/helm-toolkit/Chart.yaml b/charts/heat/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/heat/charts/helm-toolkit/Chart.yaml
+++ b/charts/heat/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/heat/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/heat/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/heat/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/heat/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/heat/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/heat/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/heat/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/heat/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/heat/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/heat/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/heat/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/heat/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/heat/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/heat/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/heat/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/heat/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/heat/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/heat/requirements.lock b/charts/heat/requirements.lock
index e346dde..43aa382 100644
--- a/charts/heat/requirements.lock
+++ b/charts/heat/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/heat/requirements.yaml b/charts/heat/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/heat/requirements.yaml
+++ b/charts/heat/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/horizon/charts/helm-toolkit/Chart.yaml b/charts/horizon/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/horizon/charts/helm-toolkit/Chart.yaml
+++ b/charts/horizon/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/horizon/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/horizon/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/horizon/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/horizon/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/horizon/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/horizon/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/horizon/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/horizon/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/horizon/requirements.lock b/charts/horizon/requirements.lock
index e346dde..43aa382 100644
--- a/charts/horizon/requirements.lock
+++ b/charts/horizon/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/horizon/requirements.yaml b/charts/horizon/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/horizon/requirements.yaml
+++ b/charts/horizon/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/keystone/charts/helm-toolkit/Chart.yaml b/charts/keystone/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/keystone/charts/helm-toolkit/Chart.yaml
+++ b/charts/keystone/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/keystone/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/keystone/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/keystone/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/keystone/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/keystone/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/keystone/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/keystone/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/keystone/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/keystone/requirements.lock b/charts/keystone/requirements.lock
index e346dde..43aa382 100644
--- a/charts/keystone/requirements.lock
+++ b/charts/keystone/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/keystone/requirements.yaml b/charts/keystone/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/keystone/requirements.yaml
+++ b/charts/keystone/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/libvirt/charts/helm-toolkit/Chart.yaml b/charts/libvirt/charts/helm-toolkit/Chart.yaml
index fd3f461..e827e99 100644
--- a/charts/libvirt/charts/helm-toolkit/Chart.yaml
+++ b/charts/libvirt/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.56
+version: 0.2.64
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/libvirt/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/libvirt/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/libvirt/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/libvirt/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/libvirt/requirements.lock b/charts/libvirt/requirements.lock
index 5cfd353..43aa382 100644
--- a/charts/libvirt/requirements.lock
+++ b/charts/libvirt/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.56
-digest: sha256:aacff4a999f7d7cc6a011849a6b040e36a8a168af72da9d0db2f00a346974769
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/libvirt/requirements.yaml b/charts/libvirt/requirements.yaml
index 2590759..ddafbfc 100644
--- a/charts/libvirt/requirements.yaml
+++ b/charts/libvirt/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.56
+  version: 0.2.64
diff --git a/charts/magnum/charts/helm-toolkit/Chart.yaml b/charts/magnum/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/magnum/charts/helm-toolkit/Chart.yaml
+++ b/charts/magnum/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/magnum/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/magnum/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/magnum/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/magnum/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/magnum/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/magnum/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/magnum/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/magnum/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/magnum/requirements.lock b/charts/magnum/requirements.lock
index e346dde..43aa382 100644
--- a/charts/magnum/requirements.lock
+++ b/charts/magnum/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/magnum/requirements.yaml b/charts/magnum/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/magnum/requirements.yaml
+++ b/charts/magnum/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/manila/charts/helm-toolkit/Chart.yaml b/charts/manila/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/manila/charts/helm-toolkit/Chart.yaml
+++ b/charts/manila/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/manila/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/manila/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/manila/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/manila/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/manila/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/manila/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/manila/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/manila/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/manila/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/manila/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/manila/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/manila/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/manila/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/manila/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/manila/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/manila/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/manila/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/manila/requirements.lock b/charts/manila/requirements.lock
index e346dde..43aa382 100644
--- a/charts/manila/requirements.lock
+++ b/charts/manila/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/manila/requirements.yaml b/charts/manila/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/manila/requirements.yaml
+++ b/charts/manila/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/memcached/charts/helm-toolkit/Chart.yaml b/charts/memcached/charts/helm-toolkit/Chart.yaml
index 404f380..e827e99 100644
--- a/charts/memcached/charts/helm-toolkit/Chart.yaml
+++ b/charts/memcached/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.52
+version: 0.2.64
diff --git a/charts/memcached/charts/helm-toolkit/requirements.lock b/charts/memcached/charts/helm-toolkit/requirements.lock
index f58d8e0..808bd94 100644
--- a/charts/memcached/charts/helm-toolkit/requirements.lock
+++ b/charts/memcached/charts/helm-toolkit/requirements.lock
@@ -1,3 +1,3 @@
 dependencies: []
 digest: sha256:643d5437104296e21d906ecb15b2c96ad278f20cfc4af53b12bb6069bd853726
-generated: "2023-03-17T21:00:03.500496699Z"
+generated: '0001-01-01T00:00:00Z'
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 4c476b2..cacb4b8 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -67,16 +67,16 @@
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         rules:
           - host: barbican
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -108,10 +108,10 @@
       metadata:
         name: barbican-namespace-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public
             hosts:
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -133,10 +133,10 @@
       metadata:
         name: barbican-cluster-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx-cluster"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx-cluster"
         tls:
           - secretName: barbican-tls-public
             hosts:
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -202,10 +202,10 @@
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public
             hosts:
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -302,12 +302,12 @@
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           cert-manager.io/issuer: ca-issuer
           certmanager.k8s.io/issuer: ca-issuer
           nginx.ingress.kubernetes.io/backend-protocol: https
           nginx.ingress.kubernetes.io/secure-backends: "true"
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public-certmanager
             hosts:
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -404,12 +404,12 @@
       metadata:
         name: barbican
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           cert-manager.io/cluster-issuer: ca-issuer
           certmanager.k8s.io/cluster-issuer: ca-issuer
           nginx.ingress.kubernetes.io/backend-protocol: https
           nginx.ingress.kubernetes.io/secure-backends: "true"
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: barbican-tls-public-certmanager
             hosts:
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -488,16 +488,16 @@
       metadata:
         name: grafana
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         rules:
           - host: grafana
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -529,10 +529,10 @@
       metadata:
         name: grafana-namespace-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx"
         tls:
           - secretName: grafana-tls-public
             hosts:
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -565,10 +565,10 @@
       metadata:
         name: grafana-cluster-fqdn
         annotations:
-          kubernetes.io/ingress.class: "nginx-cluster"
           nginx.ingress.kubernetes.io/rewrite-target: /
 
       spec:
+        ingressClassName: "nginx-cluster"
         tls:
           - secretName: grafana-tls-public
             hosts:
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -639,7 +641,6 @@
 metadata:
   name: {{ $ingressName }}
   annotations:
-    kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }}
 {{- if $certIssuer }}
     cert-manager.io/{{ $certIssuerType }}: {{ $certIssuer }}
     certmanager.k8s.io/{{ $certIssuerType }}: {{ $certIssuer }}
@@ -650,6 +651,7 @@
 {{- end }}
 {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }}
 spec:
+  ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }}
 {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "hosts" }}
 {{- if $certIssuer }}
 {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -695,9 +697,9 @@
 metadata:
   name: {{ printf "%s-%s-%s" $ingressName $ingressController "fqdn" }}
   annotations:
-    kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }}
 {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }}
 spec:
+  ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }}
 {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "host_fqdn_override" }}
 {{- if hasKey $host $endpoint }}
 {{- $endpointHost := index $host $endpoint }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl
index 4294d40..6027b95 100644
--- a/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl
@@ -133,8 +133,10 @@
 # Create DB User
 try:
     root_engine.execute(
-        "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\' IDENTIFIED BY \'{2}\' {3}".format(
-            database, user, password, mysql_x509))
+        "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format(
+            user, password, mysql_x509))
+    root_engine.execute(
+        "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))
     logger.info("Created user {0} for {1}".format(user, database))
 except:
     logger.critical("Could not create user {0} for {1}".format(user, database))
diff --git a/charts/memcached/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/memcached/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/memcached/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/memcached/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/memcached/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/memcached/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/memcached/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/memcached/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl b/charts/memcached/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl
index b99c00d..4a88dd8 100644
--- a/charts/memcached/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl
+++ b/charts/memcached/charts/helm-toolkit/templates/utils/_dependency_resolver.tpl
@@ -27,10 +27,12 @@
 {{- else if kindIs "slice" $dependencyMixinParam }}
 {{- $_ := set $envAll.Values "__deps" ( index $envAll.Values.dependencies.static $dependencyKey ) }}
 {{- range $k, $v := $dependencyMixinParam -}}
+{{- if ( index $envAll.Values.dependencies.dynamic.targeted $v ) }}
 {{- $_ := include "helm-toolkit.utils.merge" (tuple $envAll.Values.pod_dependency $envAll.Values.__deps ( index $envAll.Values.dependencies.dynamic.targeted $v $dependencyKey ) ) -}}
 {{- $_ := set $envAll.Values "__deps" $envAll.Values.pod_dependency -}}
 {{- end }}
 {{- end }}
+{{- end }}
 {{- else -}}
 {{- $_ := set $envAll.Values "pod_dependency" ( index $envAll.Values.dependencies.static $dependencyKey ) -}}
 {{- end -}}
diff --git a/charts/memcached/requirements.lock b/charts/memcached/requirements.lock
index a348b9e..43aa382 100644
--- a/charts/memcached/requirements.lock
+++ b/charts/memcached/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
-  repository: file://../helm-toolkit
-  version: 0.2.52
-digest: sha256:d7c1d04fc7525277f29dac7fc7d2996c60cb3e708f487cd2bf88a0236454f7e3
-generated: "2023-03-17T21:00:20.838477353Z"
+  repository: https://tarballs.openstack.org/openstack-helm-infra
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
+generated: '0001-01-01T00:00:00Z'
diff --git a/charts/memcached/requirements.yaml b/charts/memcached/requirements.yaml
index 84f0aff..ddafbfc 100644
--- a/charts/memcached/requirements.yaml
+++ b/charts/memcached/requirements.yaml
@@ -1,18 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
 dependencies:
-  - name: helm-toolkit
-    repository: file://../helm-toolkit
-    version: ">= 0.1.0"
-...
+- name: helm-toolkit
+  repository: https://tarballs.openstack.org/openstack-helm-infra
+  version: 0.2.64
diff --git a/charts/neutron/charts/helm-toolkit/Chart.yaml b/charts/neutron/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/neutron/charts/helm-toolkit/Chart.yaml
+++ b/charts/neutron/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/neutron/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/neutron/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/neutron/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/neutron/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/neutron/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/neutron/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/neutron/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/neutron/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/neutron/requirements.lock b/charts/neutron/requirements.lock
index e346dde..43aa382 100644
--- a/charts/neutron/requirements.lock
+++ b/charts/neutron/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/neutron/requirements.yaml b/charts/neutron/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/neutron/requirements.yaml
+++ b/charts/neutron/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/nova/charts/helm-toolkit/Chart.yaml b/charts/nova/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/nova/charts/helm-toolkit/Chart.yaml
+++ b/charts/nova/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/nova/requirements.lock b/charts/nova/requirements.lock
index e346dde..43aa382 100644
--- a/charts/nova/requirements.lock
+++ b/charts/nova/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/nova/requirements.yaml b/charts/nova/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/nova/requirements.yaml
+++ b/charts/nova/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/octavia/charts/helm-toolkit/Chart.yaml b/charts/octavia/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/octavia/charts/helm-toolkit/Chart.yaml
+++ b/charts/octavia/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/octavia/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/octavia/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/octavia/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/octavia/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/octavia/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/octavia/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/octavia/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/octavia/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/octavia/requirements.lock b/charts/octavia/requirements.lock
index e346dde..43aa382 100644
--- a/charts/octavia/requirements.lock
+++ b/charts/octavia/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/octavia/requirements.yaml b/charts/octavia/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/octavia/requirements.yaml
+++ b/charts/octavia/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/openvswitch/charts/helm-toolkit/Chart.yaml b/charts/openvswitch/charts/helm-toolkit/Chart.yaml
index fd3f461..e827e99 100644
--- a/charts/openvswitch/charts/helm-toolkit/Chart.yaml
+++ b/charts/openvswitch/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.56
+version: 0.2.64
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/openvswitch/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/openvswitch/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/openvswitch/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/openvswitch/requirements.lock b/charts/openvswitch/requirements.lock
index 5cfd353..43aa382 100644
--- a/charts/openvswitch/requirements.lock
+++ b/charts/openvswitch/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.56
-digest: sha256:aacff4a999f7d7cc6a011849a6b040e36a8a168af72da9d0db2f00a346974769
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/openvswitch/requirements.yaml b/charts/openvswitch/requirements.yaml
index 2590759..ddafbfc 100644
--- a/charts/openvswitch/requirements.yaml
+++ b/charts/openvswitch/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.56
+  version: 0.2.64
diff --git a/charts/ovn/charts/helm-toolkit/Chart.yaml b/charts/ovn/charts/helm-toolkit/Chart.yaml
index fd3f461..e827e99 100644
--- a/charts/ovn/charts/helm-toolkit/Chart.yaml
+++ b/charts/ovn/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.56
+version: 0.2.64
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/ovn/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/ovn/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/ovn/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/ovn/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/ovn/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/ovn/requirements.lock b/charts/ovn/requirements.lock
index 5cfd353..43aa382 100644
--- a/charts/ovn/requirements.lock
+++ b/charts/ovn/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.56
-digest: sha256:aacff4a999f7d7cc6a011849a6b040e36a8a168af72da9d0db2f00a346974769
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/ovn/requirements.yaml b/charts/ovn/requirements.yaml
index 2590759..ddafbfc 100644
--- a/charts/ovn/requirements.yaml
+++ b/charts/ovn/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.56
+  version: 0.2.64
diff --git a/charts/placement/charts/helm-toolkit/Chart.yaml b/charts/placement/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/placement/charts/helm-toolkit/Chart.yaml
+++ b/charts/placement/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/placement/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/placement/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/placement/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/placement/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/placement/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/placement/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/placement/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/placement/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/placement/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/placement/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/placement/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/placement/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/placement/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/placement/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/placement/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/placement/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/placement/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/placement/requirements.lock b/charts/placement/requirements.lock
index e346dde..43aa382 100644
--- a/charts/placement/requirements.lock
+++ b/charts/placement/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/placement/requirements.yaml b/charts/placement/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/placement/requirements.yaml
+++ b/charts/placement/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/senlin/charts/helm-toolkit/Chart.yaml b/charts/senlin/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/senlin/charts/helm-toolkit/Chart.yaml
+++ b/charts/senlin/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/senlin/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/senlin/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/senlin/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/senlin/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/senlin/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/senlin/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/senlin/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/senlin/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/senlin/requirements.lock b/charts/senlin/requirements.lock
index e346dde..43aa382 100644
--- a/charts/senlin/requirements.lock
+++ b/charts/senlin/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/senlin/requirements.yaml b/charts/senlin/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/senlin/requirements.yaml
+++ b/charts/senlin/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64
diff --git a/charts/tempest/charts/helm-toolkit/Chart.yaml b/charts/tempest/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/tempest/charts/helm-toolkit/Chart.yaml
+++ b/charts/tempest/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/tempest/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/tempest/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/tempest/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/tempest/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/tempest/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/tempest/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/tempest/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/tempest/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/tempest/requirements.lock b/charts/tempest/requirements.lock
index e346dde..43aa382 100644
--- a/charts/tempest/requirements.lock
+++ b/charts/tempest/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/tempest/requirements.yaml b/charts/tempest/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/tempest/requirements.yaml
+++ b/charts/tempest/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64