diff --git a/charts/nova/charts/helm-toolkit/Chart.yaml b/charts/nova/charts/helm-toolkit/Chart.yaml
index 1ee9758..e827e99 100644
--- a/charts/nova/charts/helm-toolkit/Chart.yaml
+++ b/charts/nova/charts/helm-toolkit/Chart.yaml
@@ -9,4 +9,4 @@
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.55
+version: 0.2.64
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl
index 972e429..cacb4b8 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl
@@ -59,7 +59,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -76,7 +76,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -86,7 +86,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -96,7 +96,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -121,7 +121,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -146,7 +146,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -194,7 +194,7 @@
               default: 9311
               public: 80
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -217,7 +217,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -227,7 +227,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -237,7 +237,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -294,7 +294,7 @@
                 name: ca-issuer
                 kind: Issuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -319,7 +319,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -329,7 +329,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -339,7 +339,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -396,7 +396,7 @@
                 name: ca-issuer
                 kind: ClusterIssuer
     usage: |
-      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}}
+      {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}}
     return: |
       ---
       apiVersion: networking.k8s.io/v1
@@ -421,7 +421,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -431,7 +431,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -441,7 +441,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: barbican-api
@@ -479,7 +479,7 @@
             grafana:
               public: grafana-tls-public
     usage: |
-      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}}
+      {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}}
       {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
     return: |
       ---
@@ -497,7 +497,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -507,7 +507,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -517,7 +517,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -543,7 +543,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -553,7 +553,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -579,7 +579,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -589,7 +589,7 @@
             http:
               paths:
                 - path: /
-                  pathType: ImplementationSpecific
+                  pathType: Prefix
                   backend:
                     service:
                       name: grafana-dashboard
@@ -602,11 +602,12 @@
 {{- $vHost := index . "vHost" -}}
 {{- $backendName := index . "backendName" -}}
 {{- $backendPort := index . "backendPort" -}}
+{{- $pathType := index . "pathType" -}}
 - host: {{ $vHost }}
   http:
     paths:
       - path: /
-        pathType: ImplementationSpecific
+        pathType: {{ $pathType }}
         backend:
           service:
             name: {{ $backendName }}
@@ -624,6 +625,7 @@
 {{- $backendServiceType := index . "backendServiceType" -}}
 {{- $backendPort := index . "backendPort" -}}
 {{- $endpoint := index . "endpoint" | default "public" -}}
+{{- $pathType := index . "pathType" | default "Prefix" -}}
 {{- $certIssuer := index . "certIssuer" | default "" -}}
 {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@@ -681,7 +683,7 @@
 {{- end }}
   rules:
 {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
-{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }}
@@ -719,7 +721,7 @@
 {{- end }}
   rules:
 {{- range $vHost := $vHosts }}
-{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }}
+{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }}
 {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
index 5d98c8b..6b77004 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl
@@ -51,6 +51,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 62ed119..2b7ff2c 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -54,6 +54,7 @@
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 745e8da..b8a1dce 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 24d2496..4696c88 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -49,6 +49,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 3a7df7f..d69c9e6 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
index a109e3c..9604c63 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl
@@ -52,6 +52,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
index 905eb71..58dcdc5 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
@@ -74,6 +74,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
index 6982064..2cfadaf 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl
@@ -42,6 +42,7 @@
 {{ toYaml $jobLabels | indent 4 }}
 {{- end }}
   annotations:
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index 29cb993..b5fdc09 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -49,6 +49,7 @@
 {{- end }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
index 50d9af5..77d1a71 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
@@ -47,6 +47,7 @@
   annotations:
     "helm.sh/hook-delete-policy": before-hook-creation
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}}
 {{- if $jobAnnotations }}
 {{ toYaml $jobAnnotations | indent 4 }}
 {{- end }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
index 4854bb1..7ad505b 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a authenticating a registry with a secret
 examples:
   - values: |
+      annotations:
+        secret:
+          oci_image_registry:
+            {{ $serviceName }}:
+              custom.tld/key: "value"
       secrets:
         oci_image_registry:
           {{ $serviceName }}: {{ $keyName }}
@@ -36,30 +41,8 @@
     kind: Secret
     metadata:
       name: {{ $secretName }}
-    type: kubernetes.io/dockerconfigjson
-    data:
-      dockerconfigjson: {{ $dockerAuth }}
-
-  - values: |
-      secrets:
-        oci_image_registry:
-          {{ $serviceName }}: {{ $keyName }}
-      endpoints:
-        oci_image_registry:
-          name: oci-image-registry
-          auth:
-            enabled: true
-             {{ $serviceName }}:
-                name: {{ $userName }}
-                password: {{ $password }}
-  usage: |
-    {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}}
-  return: |
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: {{ $secretName }}
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/dockerconfigjson
     data:
       dockerconfigjson: {{ $dockerAuth }}
@@ -87,6 +70,8 @@
 kind: Secret
 metadata:
   name: {{ $secretName }}
+  annotations:
+{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ $dockerAuth }}
diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
index 24a7045..c800340 100644
--- a/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl
@@ -17,6 +17,11 @@
   Creates a manifest for a services public tls secret
 examples:
   - values: |
+      annotations:
+        secret:
+          tls:
+            key_manager_api_public:
+              custom.tld/key: "value"
       secrets:
         tls:
           key_manager:
@@ -41,6 +46,8 @@
     kind: Secret
     metadata:
       name: barbican-tls-public
+      annotations:
+        custom.tld/key: "value"
     type: kubernetes.io/tls
     data:
       tls.key: Rk9PLUtFWQo=
@@ -88,11 +95,15 @@
 {{- if kindIs "map" $endpointHost }}
 {{- if hasKey $endpointHost "tls" }}
 {{- if and $endpointHost.tls.key $endpointHost.tls.crt }}
+
+{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
+  annotations:
+{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $endpointHost.tls.key | b64enc }}
diff --git a/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
index 3963bd4..695cb2e 100644
--- a/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/charts/nova/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@@ -49,6 +49,13 @@
 #                                          A random number between min and max delay is generated
 #                                          to set the delay.
 #
+#         RGW backup throttle limits variables:
+#           export THROTTLE_BACKUPS_ENABLED   Boolean variableto control backup functionality
+#           export THROTTLE_LIMIT             Number of simultaneous RGW upload sessions
+#           export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned
+#           export THROTTLE_RETRY_AFTER       Time in seconds to wait before retry
+#           export THROTTLE_CONTAINER_NAME    Name of RGW container to place flag falies into
+#
 # The database-specific functions that need to be implemented are:
 #   dump_databases_to_directory <directory> <err_logfile> [scope]
 #       where:
@@ -84,8 +91,10 @@
 #      specified by the "LOCAL_DAYS_TO_KEEP" variable.
 #   4) Removing remote backup tarballs (from the remote gateway) which are older
 #      than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable.
+#   5) Controlling remote storage gateway load from client side and throttling it
+#      by using a dedicated RGW container to store flag files defining upload session
+#      in progress
 #
-
 # Note: not using set -e in this script because more elaborate error handling
 # is needed.
 
@@ -95,7 +104,7 @@
   log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 log_verify_backup_exit() {
@@ -104,7 +113,7 @@
   log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
   rm -f $ERR_LOG_FILE
   # rm -rf $TMP_DIR
-  exit $ERRCODE
+  exit 0
 }
 
 
@@ -218,6 +227,113 @@
   echo "Sleeping for ${DELAY} seconds to spread the load in time..."
   sleep ${DELAY}
 
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove Quotes from the constants which were added due to reading
+    # from secret.
+    export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g')
+    export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g')
+    export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g')
+    export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g')
+
+    # load balance delay
+    RESULT=$(openstack container list 2>&1)
+
+    if [[ $? -eq 0 ]]; then
+      echo $RESULT | grep $THROTTLE_CONTAINER_NAME
+      if [[ $? -ne 0 ]]; then
+        # Find the swift URL from the keystone endpoint list
+        SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}')
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog."
+          return 2
+        fi
+
+        # Get a token from keystone
+        TOKEN=$(openstack token issue -f value -c id)
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to get  keystone token."
+          return 2
+        fi
+
+        # Create the container
+        RES_FILE=$(mktemp -p /tmp)
+        curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \
+            -H "X-Auth-Token: ${TOKEN}" \
+            -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE
+
+        if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}"
+          cat $RES_FILE
+          rm -f $RES_FILE
+          return 2
+        fi
+        rm -f $RES_FILE
+
+        swift stat $THROTTLE_CONTAINER_NAME
+        if [[ $? -ne 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation."
+          return 2
+        fi
+      fi
+    else
+      echo $RESULT | grep -E "HTTP 401|HTTP 403"
+      if [[ $? -eq 0 ]]; then
+        log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}"
+        return 1
+      else
+        echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50"
+        if [[ $? -eq 0 ]]; then
+          log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}"
+          # In this case, keystone or the site/node may be temporarily down.
+          # Return slightly different error code so the calling code can retry
+          return 2
+        else
+          log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}"
+          return 1
+        fi
+      fi
+    fi
+
+    NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+    log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]]
+    do
+      log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!"
+      log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...."
+      sleep ${THROTTLE_RETRY_AFTER}
+      NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l)
+      log INFO  "${DB_NAME}_backup"  "There are ${NUMBER_OF_SESSIONS} remote sessions right now."
+    done
+
+    # Create a lock file in THROTTLE_CONTAINER
+    THROTTLE_FILEPATH=$(mktemp -d)
+    THROTTLE_FILE=${CONTAINER_NAME}.lock
+    date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE
+
+    # Create an object to store the file
+    openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!"
+      return 2
+    fi
+
+    swift post  $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}"
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!"
+      return 2
+    fi
+    openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation."
+      return 2
+    fi
+  fi
+
+  #---------------------------------------------------------------------------
+
   # Create an object to store the file
   openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE
   if [[ $? -ne 0 ]]; then
@@ -243,7 +359,25 @@
       log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
       return 2
   fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}
+
+  #---------------------------------------------------------------------------
+  # Remote backup throttling
+  export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g')
+  if $THROTTLE_BACKUPS_ENABLED; then
+    # Remove flag file
+    # Delete an object to remove the flag file
+    openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE
+    if [[ $? -ne 0 ]]; then
+      log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}"
+      return 0
+    else
+      log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed."
+    fi
+    rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE}
+  fi
+
+  #---------------------------------------------------------------------------
 
   log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
   return 0
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
new file mode 100644
index 0000000..fc42614
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the job spec of a component.
+examples:
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        job:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          keystone_domain_manage:
+            another.tld/foo: "bar"
+          keystone_bootstrap:
+    usage: |
+      {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_job_annotations" -}}
+{{- $envAll := index . 1 -}}
+{{- $component := index . 0 | replace "-" "_" -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "job") -}}
+{{- $annotationsMap := $envAll.Values.annotations.job -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
new file mode 100644
index 0000000..ecff6e9
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl
@@ -0,0 +1,76 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the pod spec of a component.
+examples:
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        pod:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          nova_compute:
+            another.tld/foo: "bar"
+          nova_api:
+    usage: |
+      {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_pod_annotations" -}}
+{{- $component := index . 0 -}}
+{{- $envAll := index . 1 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "pod") -}}
+{{- $annotationsMap := $envAll.Values.annotations.pod -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $annotationsMap "default" ) -}}
+{{- $defaultAnnotations = $annotationsMap.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
new file mode 100644
index 0000000..19c4380
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl
@@ -0,0 +1,81 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+abstract: |
+  Adds custom annotations to the secret spec of a component.
+examples:
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      another.tld/foo: bar
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+  - values: |
+      annotations:
+        secret:
+          default:
+            custom.tld/key: "value"
+            custom.tld/key2: "value2"
+          identity:
+            admin:
+              another.tld/foo: "bar"
+          oslo_db:
+            admin:
+    usage: |
+      {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }}
+    return: |
+      custom.tld/key: "value"
+      custom.tld/key2: "value2"
+*/}}
+
+{{- define "helm-toolkit.snippets.custom_secret_annotations" -}}
+{{- $secretType := index . 0 -}}
+{{- $userClass := index . 1 | replace "-" "_" -}}
+{{- $envAll := index . 2 -}}
+{{- if (hasKey $envAll.Values "annotations") -}}
+{{- if (hasKey $envAll.Values.annotations "secret") -}}
+{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}}
+{{- $defaultAnnotations := dict -}}
+{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}}
+{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}}
+{{- end -}}
+{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}}
+{{- if (not (empty $annotations)) -}}
+{{- toYaml $annotations -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
new file mode 100644
index 0000000..08521e0
--- /dev/null
+++ b/charts/nova/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
diff --git a/charts/nova/requirements.lock b/charts/nova/requirements.lock
index e346dde..43aa382 100644
--- a/charts/nova/requirements.lock
+++ b/charts/nova/requirements.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
-digest: sha256:92b4001062b365cd9687eb926780c7f64e17e488262c1167339ade0fe600b368
+  version: 0.2.64
+digest: sha256:4c00b9bfa1d3dc0426a82ec22f51b440e838c55cbd1f81dbf7de5b28471f6141
 generated: '0001-01-01T00:00:00Z'
diff --git a/charts/nova/requirements.yaml b/charts/nova/requirements.yaml
index 6ab539f..ddafbfc 100644
--- a/charts/nova/requirements.yaml
+++ b/charts/nova/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - name: helm-toolkit
   repository: https://tarballs.openstack.org/openstack-helm-infra
-  version: 0.2.55
+  version: 0.2.64