Switch to new pinimages
Change-Id: I85694d9b108abc6603ccf5bf82df2371d22a3249
diff --git a/build/pin-images.py b/build/pin-images.py
deleted file mode 100755
index fbc8f2c..0000000
--- a/build/pin-images.py
+++ /dev/null
@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import functools
-
-import requests
-from docker_image import reference
-from oslo_config import cfg
-from oslo_log import log as logging
-from ruyaml import YAML
-
-LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
-
-SKIP_IMAGE_LIST = ["secretgen_controller"]
-
-
-def get_digest(image_ref, token=None):
- url = f"https://{image_ref.domain()}/v2/{image_ref.path()}/manifests/{image_ref['tag']}"
-
- headers = {}
- if token:
- headers["Authorization"] = f"Bearer {token}"
- else:
- r = requests.get(url, timeout=5, verify=False)
- auth_header = r.headers.get("Www-Authenticate")
- if auth_header:
- realm = auth_header.split(",")[0].split("=")[1].strip('"')
-
- r = requests.get(
- realm,
- timeout=5,
- verify=False,
- params={"scope": f"repository:{image_ref.path()}:pull"},
- )
- r.raise_for_status()
-
- headers["Authorization"] = f"Bearer {r.json()['token']}"
-
- try:
- headers["Accept"] = "application/vnd.docker.distribution.manifest.v2+json"
-
- r = requests.get(
- f"https://{image_ref.domain()}/v2/{image_ref.path()}/manifests/{image_ref['tag']}",
- timeout=5,
- verify=False,
- headers=headers,
- )
- r.raise_for_status()
- return r.headers["Docker-Content-Digest"]
- except requests.exceptions.HTTPError:
- headers["Accept"] = "application/vnd.oci.image.index.v1+json"
-
- r = requests.get(
- f"https://{image_ref.domain()}/v2/{image_ref.path()}/manifests/{image_ref['tag']}",
- timeout=5,
- verify=False,
- headers=headers,
- )
- r.raise_for_status()
- return r.headers["Docker-Content-Digest"]
-
-
-@functools.cache
-def get_pinned_image(image_src):
- image_ref = reference.Reference.parse(image_src)
- if image_ref.domain() != "harbor.atmosphere.dev":
- try:
- image_ref = reference.Reference.parse("harbor.atmosphere.dev/" + image_src)
- except Exception:
- LOG.warn(f"failed to parse image path {image_src}")
-
- if (
- image_ref.domain() == "registry.atmosphere.dev"
- or image_ref.domain() == "harbor.atmosphere.dev"
- ):
- # Get token for docker.io
- r = requests.get(
- "https://harbor.atmosphere.dev/service/token",
- timeout=5,
- params={
- "service": "harbor-registry",
- "scope": f"repository:{image_ref.path()}:pull",
- },
- )
- r.raise_for_status()
- token = r.json()["token"]
-
- digest = get_digest(image_ref, token=token)
- elif image_ref.domain() == "quay.io":
- r = requests.get(
- f"https://quay.io/api/v1/repository/{image_ref.path()}/tag/",
- timeout=5,
- params={"specificTag": image_ref["tag"]},
- )
- r.raise_for_status()
- digest = r.json()["tags"][0]["manifest_digest"]
- elif image_ref.domain() == "docker.io":
- # Get token for docker.io
- r = requests.get(
- "https://auth.docker.io/token",
- timeout=5,
- params={
- "service": "registry.docker.io",
- "scope": f"repository:{image_ref.path()}:pull",
- },
- )
- r.raise_for_status()
- token = r.json()["token"]
-
- r = requests.get(
- f"https://registry-1.docker.io/v2/{image_ref.path()}/manifests/{image_ref['tag']}",
- timeout=5,
- headers={
- "Accept": "application/vnd.docker.distribution.manifest.v2+json",
- "Authorization": f"Bearer {token}",
- },
- )
- r.raise_for_status()
- digest = r.headers["Docker-Content-Digest"]
- elif image_ref.domain() == "ghcr.io":
- # Get token for docker.io
- r = requests.get(
- "https://ghcr.io/token",
- timeout=5,
- params={
- "service": "ghcr.io",
- "scope": f"repository:{image_ref.path()}:pull",
- },
- )
- r.raise_for_status()
- token = r.json()["token"]
-
- digest = get_digest(image_ref, token=token)
- else:
- digest = get_digest(image_ref)
-
- original_ref = reference.Reference.parse(image_src)
- return (
- f"{original_ref.domain()}/{original_ref.path()}:{original_ref['tag']}@{digest}"
- )
-
-
-def main():
- logging.register_options(CONF)
- logging.setup(CONF, "atmosphere-bump-images")
-
- parser = argparse.ArgumentParser("bump-images")
- parser.add_argument(
- "src", help="Path for default values file", type=argparse.FileType("r")
- )
- parser.add_argument(
- "dst", help="Path for output file", type=argparse.FileType("r+")
- )
-
- args = parser.parse_args()
-
- yaml = YAML(typ="rt")
- data = yaml.load(args.src)
-
- for image in data["_atmosphere_images"]:
- if image in SKIP_IMAGE_LIST:
- continue
-
- image_src = (
- data["_atmosphere_images"][image]
- .replace("{{ atmosphere_release }}", data["atmosphere_release"])
- .replace("{{ atmosphere_image_prefix }}", "")
- )
- pinned_image = get_pinned_image(image_src).replace(
- "harbor.atmosphere.dev", "registry.atmosphere.dev"
- )
-
- LOG.info("Pinning image %s from %s to %s", image, image_src, pinned_image)
- data["_atmosphere_images"][image] = "{{ atmosphere_image_prefix }}%s" % (
- pinned_image,
- )
-
- yaml.dump(data, args.dst)
-
-
-if __name__ == "__main__":
- main()
diff --git a/cmd/pinimages/pinimages.go b/cmd/pinimages/pinimages.go
new file mode 100644
index 0000000..c19841e
--- /dev/null
+++ b/cmd/pinimages/pinimages.go
@@ -0,0 +1,197 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+
+ "github.com/containers/image/v5/docker"
+ "github.com/containers/image/v5/manifest"
+ "github.com/containers/image/v5/types"
+ "github.com/goccy/go-yaml"
+ "github.com/goccy/go-yaml/ast"
+ "github.com/goccy/go-yaml/parser"
+ "github.com/opencontainers/go-digest"
+ log "github.com/sirupsen/logrus"
+ "golang.org/x/sync/singleflight"
+)
+
+var digestGroup singleflight.Group
+
+// GetImageDigest fetches the digest for a given image reference.
+func GetImageDigest(ctx context.Context, reference string) (digest.Digest, error) {
+ ref, err := docker.ParseReference(reference)
+ if err != nil {
+ return "", fmt.Errorf("error parsing reference '%s': %w", reference, err)
+ }
+
+ sysCtx := &types.SystemContext{}
+ imageSource, err := ref.NewImageSource(ctx, sysCtx)
+ if err != nil {
+ return "", fmt.Errorf("error creating image source for '%s': %w", reference, err)
+ }
+ defer imageSource.Close()
+
+ rawManifest, _, err := imageSource.GetManifest(ctx, nil)
+ if err != nil {
+ return "", fmt.Errorf("error getting manifest for '%s': %w", reference, err)
+ }
+
+ dgst, err := manifest.Digest(rawManifest)
+ if err != nil {
+ return "", fmt.Errorf("error getting digest for '%s': %w", reference, err)
+ }
+
+ return dgst, nil
+}
+
+// GetImageNameToPull normalizes the image name by replacing variables and adding necessary prefixes.
+func GetImageNameToPull(image string, release string) string {
+ // Replace Jinja2 variables with actual values
+ image = strings.ReplaceAll(image, "{{ atmosphere_image_prefix }}", "")
+ image = strings.ReplaceAll(image, "{{ atmosphere_release }}", release)
+
+ // Add mirror if the image is not hosted with us
+ if !strings.HasPrefix(image, "registry.atmosphere.dev") {
+ image = fmt.Sprintf("harbor.atmosphere.dev/%s", image)
+ }
+
+ // Switch out of the CDN since we are in CI
+ if strings.HasPrefix(image, "registry.atmosphere.dev") {
+ image = strings.ReplaceAll(image, "registry.atmosphere.dev", "harbor.atmosphere.dev")
+ }
+
+ return image
+}
+
+// AppendDigestToImage appends the digest to the original image reference.
+func AppendDigestToImage(image string, dgst digest.Digest) string {
+ if strings.Contains(image, "@") {
+ // Replace existing digest if present
+ parts := strings.Split(image, "@")
+ return parts[0] + "@" + dgst.String()
+ }
+ // Append digest
+ return image + "@" + dgst.String()
+}
+
+func main() {
+ varsFilePath := "roles/defaults/vars/main.yml"
+
+ file, err := parser.ParseFile(varsFilePath, parser.ParseComments)
+ if err != nil {
+ log.WithError(err).Fatal("error parsing yaml file")
+ }
+
+ if len(file.Docs) != 1 {
+ log.Fatal("expected exactly one yaml document")
+ }
+
+ doc := file.Docs[0]
+ body := doc.Body.(*ast.MappingNode)
+
+ var release string
+ var images *ast.MappingNode
+
+ for _, item := range body.Values {
+ switch item.Key.(*ast.StringNode).Value {
+ case "atmosphere_release":
+ release = item.Value.(*ast.StringNode).Value
+ case "_atmosphere_images":
+ images = item.Value.(*ast.MappingNode)
+ }
+ }
+
+ if release == "" {
+ log.Fatalf("atmosphere_release not found")
+ }
+
+ if images == nil {
+ log.Fatalf("_atmosphere_images not found")
+ }
+
+ type imageInfo struct {
+ Key string
+ Value string
+ Normalized string
+ Digest digest.Digest
+ }
+
+ var imageInfos []imageInfo
+ uniqueImages := make(map[string][]int)
+
+ for i, item := range images.Values {
+ normalized := GetImageNameToPull(item.Value.(*ast.StringNode).Value, release)
+ info := imageInfo{
+ Key: item.Key.(*ast.StringNode).Value,
+ Value: item.Value.(*ast.StringNode).Value,
+ Normalized: normalized,
+ }
+ imageInfos = append(imageInfos, info)
+ uniqueImages[normalized] = append(uniqueImages[normalized], i)
+ }
+
+ digestMap := make(map[string]digest.Digest)
+ var mapMutex sync.Mutex
+ var wg sync.WaitGroup
+
+ for normImg := range uniqueImages {
+ wg.Add(1)
+ go func(normImg string) {
+ defer wg.Done()
+
+ result, err, _ := digestGroup.Do(normImg, func() (interface{}, error) {
+ dgst, err := GetImageDigest(context.TODO(), "//"+normImg)
+ if err != nil {
+ return nil, err
+ }
+ return dgst, nil
+ })
+
+ if err != nil {
+ log.WithError(err).WithFields(log.Fields{
+ "image": normImg,
+ }).Error("Error fetching digest")
+ return
+ }
+
+ dgst := result.(digest.Digest)
+
+ mapMutex.Lock()
+ digestMap[normImg] = dgst
+ mapMutex.Unlock()
+
+ log.WithFields(log.Fields{
+ "image": normImg,
+ "digest": dgst,
+ }).Info("Fetched image digest")
+ }(normImg)
+ }
+
+ wg.Wait()
+
+ // Update the image references with digests
+ for normImg, indices := range uniqueImages {
+ dgst, exists := digestMap[normImg]
+ if !exists {
+ log.WithField("image", normImg).Error("Digest not found, skipping update")
+ continue
+ }
+ for _, idx := range indices {
+ updatedImage, err := yaml.ValueToNode(AppendDigestToImage(imageInfos[idx].Value, dgst))
+ if err != nil {
+ log.WithError(err).Fatal("error converting value to node")
+ }
+
+ images.Values[idx].Value = updatedImage
+ }
+ }
+
+ if err := os.WriteFile(varsFilePath, []byte(file.String()), 0644); err != nil {
+ log.WithError(err).Fatal("error writing updated yaml file")
+ }
+
+ log.Info("Successfully updated YAML file with image digests")
+}
diff --git a/cmd/pinimages/pinimages_test.go b/cmd/pinimages/pinimages_test.go
new file mode 100644
index 0000000..3c480d2
--- /dev/null
+++ b/cmd/pinimages/pinimages_test.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetImageNameToPull(t *testing.T) {
+ tests := []struct {
+ image string
+ release string
+ want string
+ }{
+ {
+ image: "{{ atmosphere_image_prefix }}quay.io/ceph/ceph:v18.2.2",
+ release: "2024.1",
+ want: "harbor.atmosphere.dev/quay.io/ceph/ceph:v18.2.2",
+ },
+ {
+ image: "{{ atmosphere_image_prefix }}registry.atmosphere.dev/library/glance:{{ atmosphere_release }}",
+ release: "2024.1",
+ want: "harbor.atmosphere.dev/library/glance:2024.1",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.image, func(t *testing.T) {
+ got := GetImageNameToPull(tt.image, tt.release)
+
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
diff --git a/go.mod b/go.mod
index 9917561..3b63216 100644
--- a/go.mod
+++ b/go.mod
@@ -7,8 +7,9 @@
require (
github.com/containers/image/v5 v5.30.1
github.com/erikgeiser/promptkit v0.9.0
- github.com/goccy/go-yaml v1.11.3
+ github.com/goccy/go-yaml v1.15.15
github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
+ github.com/opencontainers/go-digest v1.0.0
github.com/percona/percona-xtradb-cluster-operator v1.14.0
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/common v0.55.0
@@ -18,6 +19,7 @@
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
github.com/yannh/kubeconform v0.6.4
+ golang.org/x/sync v0.8.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
gorm.io/driver/mysql v1.5.6
@@ -141,7 +143,6 @@
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
- github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
@@ -172,12 +173,10 @@
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
- golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/term v0.24.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.5.0 // indirect
- golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
diff --git a/go.sum b/go.sum
index caac830..3921d08 100644
--- a/go.sum
+++ b/go.sum
@@ -171,12 +171,6 @@
github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
github.com/go-openapi/swag v0.22.10 h1:4y86NVn7Z2yYd6pfS4Z+Nyh3aAUL3Nul+LMbhFKy0gA=
github.com/go-openapi/swag v0.22.10/go.mod h1:Cnn8BYtRlx6BNE3DPN86f/xkapGIcLWzh3CLEb4C1jI=
-github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
-github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
-github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
-github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
-github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
@@ -186,8 +180,8 @@
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
-github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
+github.com/goccy/go-yaml v1.15.15 h1:5turdzAlutS2Q7/QR/9R99Z1K0J00qDb4T0pHJcZ5ew=
+github.com/goccy/go-yaml v1.15.15/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -288,8 +282,6 @@
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
-github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
-github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
@@ -559,8 +551,6 @@
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
diff --git a/tox.ini b/tox.ini
index 0673da3..d00e8e4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,16 +20,6 @@
commands =
{posargs}
-[testenv:pin-digests]
-skip_install = true
-deps =
- docker-image-py>=0.1.12
- oslo_config
- oslo_log
- ruyaml
-commands =
- python3 {toxinidir}/build/pin-images.py roles/defaults/vars/main.yml roles/defaults/vars/main.yml
-
[testenv:linters]
skipsdist = True
deps =
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index afd3222..29a479d 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -49,7 +49,6 @@
name: atmosphere-build-collection
parent: build-ansible-collection
pre-run:
- - zuul.d/playbooks/common/switch-to-atmosphere-mirror.yml
- zuul.d/playbooks/build-collection/pre.yml
irrelevant-files:
- ^doc/
diff --git a/zuul.d/playbooks/build-collection/pre.yml b/zuul.d/playbooks/build-collection/pre.yml
index e57d724..450407b 100644
--- a/zuul.d/playbooks/build-collection/pre.yml
+++ b/zuul.d/playbooks/build-collection/pre.yml
@@ -20,6 +20,13 @@
- name: Prepare for collection build
hosts: all
+ pre_tasks:
+ - name: Ensure "go" is installed
+ ansible.builtin.include_role:
+ name: ensure-go
+ vars:
+ go_version: "1.22.3"
+
tasks:
- name: Find all roles
find:
@@ -45,9 +52,9 @@
- name: Pin all image digests
ansible.builtin.include_role:
- name: tox
+ name: go
vars:
- tox_envlist: pin-digests
+ go_command: run cmd/pinimages/pinimages.go
- name: Print out the new image manifest file
ansible.builtin.command: |