[ATMOSPHERE-332] Switch to "chart-vendor" (#1728)
This is an automated cherry-pick of #1719
/assign mnaser
diff --git a/build/sync-charts.py b/build/sync-charts.py
deleted file mode 100644
index a2be651..0000000
--- a/build/sync-charts.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# Copyright (c) 2024 VEXXHOST, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import asyncio
-import os
-import pathlib
-import textwrap
-from datetime import datetime, timezone
-
-import aiopath
-import aioshutil
-import platformdirs
-from asynctempfile import NamedTemporaryFile
-from gerrit import GerritClient
-from pydantic import BaseModel, HttpUrl, PrivateAttr
-from pydantic_yaml import parse_yaml_file_as, to_yaml_file
-
-
-class ChartRepository(BaseModel):
- url: HttpUrl
-
- @property
- def name(self):
- return self.url.host.replace(".", "-") + self.url.path.replace("/", "-")
-
-
-class ChartPatches(BaseModel):
- gerrit: dict[str, list[int]] = {}
-
-
-class ChartDependency(BaseModel):
- name: str
- repository: HttpUrl
- version: str
-
-
-class ChartRequirements(BaseModel):
- dependencies: list[ChartDependency] = []
-
-
-class ChartLock(BaseModel):
- dependencies: list[ChartDependency] = []
- digest: str
- generated: datetime
-
- class Config:
- json_encoders = {
- "generated": lambda dt: dt.isoformat(),
- }
-
-
-class Chart(BaseModel):
- name: str
- version: str
- repository: ChartRepository
- dependencies: list[ChartDependency] = []
- patches: ChartPatches = ChartPatches()
-
-
-async def patch(input: bytes, path: aiopath.AsyncPath):
- async with NamedTemporaryFile() as temp_file:
- await temp_file.write(
- textwrap.dedent(
- f"""\
- {path.name}/*
- """
- )
- .strip()
- .encode()
- )
- await temp_file.flush()
-
- proc = await asyncio.create_subprocess_shell(
- f"filterdiff -p1 -I {temp_file.name}",
- stdin=asyncio.subprocess.PIPE,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- stdout, stderr = await proc.communicate(input=input)
- if proc.returncode != 0:
- raise Exception(stderr)
-
- async with NamedTemporaryFile() as temp_file:
- await temp_file.write(
- textwrap.dedent(
- f"""\
- {path.name}/Chart.yaml
- {path.name}/values_overrides/*
- """
- )
- .strip()
- .encode()
- )
- await temp_file.flush()
-
- proc = await asyncio.create_subprocess_shell(
- f"filterdiff -p1 -X {temp_file.name}",
- stdin=asyncio.subprocess.PIPE,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- stdout, stderr = await proc.communicate(input=stdout)
- if proc.returncode != 0:
- raise Exception(stderr)
-
- proc = await asyncio.create_subprocess_shell(
- f"patch -p2 -d {path} -E",
- stdin=asyncio.subprocess.PIPE,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- stdout, stderr = await proc.communicate(input=stdout)
- if proc.returncode != 0:
- raise Exception(stdout)
-
-
-class Config(BaseModel):
- charts: list[Chart]
-
- _workspace: pathlib.Path = PrivateAttr(
- default=pathlib.Path(
- platformdirs.user_cache_dir("atmosphere-sync-charts", "vexxhost")
- )
- )
-
- @property
- def repositories(self):
- repositories = []
-
- for chart in self.charts:
- if chart.repository in repositories:
- continue
- repositories.append(chart.repository)
-
- return repositories
-
- async def _helm(self, args: list[str]):
- proc = await asyncio.create_subprocess_shell(
- f"helm {' '.join(args)}",
- env={**dict(os.environ), **{"HOME": str(self._workspace)}},
- )
- await proc.communicate()
- if proc.returncode != 0:
- raise Exception(f"helm {' '.join(args)} failed")
-
- async def _fetch_chart(self, chart: Chart, path="charts"):
- charts_path: aiopath.AsyncPath = aiopath.AsyncPath(path)
- chart_path = charts_path / chart.name
-
- try:
- await aioshutil.rmtree(f"{path}/{chart.name}-{chart.version}")
- except FileNotFoundError:
- pass
-
- try:
- try:
- os.rename(
- f"{path}/{chart.name}", f"{path}/{chart.name}-{chart.version}"
- )
- except FileNotFoundError:
- pass
-
- await self._helm(
- [
- "fetch",
- "--untar",
- f"--destination={path}",
- f"{chart.repository.name}/{chart.name}",
- f"--version={chart.version}",
- ]
- )
- except Exception:
- os.rename(f"{path}/{chart.name}-{chart.version}", f"{path}/{chart.name}")
- raise
-
- try:
- await aioshutil.rmtree(f"{path}/{chart.name}-{chart.version}")
- except FileNotFoundError:
- pass
-
- if chart.dependencies:
- requirements = ChartRequirements(dependencies=chart.dependencies)
- to_yaml_file(f"{path}/{chart.name}/requirements.yaml", requirements)
-
- await asyncio.gather(
- *[
- aioshutil.rmtree(f"{path}/{chart.name}/charts/{req.name}")
- for req in chart.dependencies
- ]
- )
-
- await self._helm(
- ["dependency", "update", "--skip-refresh", f"{path}/{chart.name}"]
- )
-
- await asyncio.gather(
- *[
- aioshutil.unpack_archive(
- f"{path}/{chart.name}/charts/{req.name}-{req.version}.tgz",
- f"{path}/{chart.name}/charts",
- )
- for req in chart.dependencies
- ]
- )
-
- await asyncio.gather(
- *[
- (chart_path / "charts" / f"{req.name}-{req.version}.tgz").unlink()
- for req in chart.dependencies
- ]
- )
-
- for req in chart.dependencies:
- lock = parse_yaml_file_as(
- ChartLock,
- f"{path}/{chart.name}/charts/{req.name}/requirements.lock",
- )
- lock.generated = datetime.min.replace(tzinfo=timezone.utc)
- to_yaml_file(
- f"{path}/{chart.name}/charts/{req.name}/requirements.lock", lock
- )
-
- # Reset the generated time in the lock file to make things reproducible
- lock = parse_yaml_file_as(
- ChartLock, f"{path}/{chart.name}/requirements.lock"
- )
- lock.generated = datetime.min.replace(tzinfo=timezone.utc)
- to_yaml_file(f"{path}/{chart.name}/requirements.lock", lock)
-
- for gerrit, changes in chart.patches.gerrit.items():
- client = GerritClient(base_url=f"https://{gerrit}")
-
- for change_id in changes:
- change = client.changes.get(change_id)
- gerrit_patch = change.get_revision().get_patch(decode=True)
- await patch(input=gerrit_patch.encode(), path=chart_path)
-
- patches_path = charts_path / "patches" / chart.name
- if await patches_path.exists():
- patch_paths = sorted(
- [patch_path async for patch_path in patches_path.glob("*.patch")]
- )
- for patch_path in patch_paths:
- async with patch_path.open(mode="rb") as patch_file:
- patch_data = await patch_file.read()
- await patch(input=patch_data, path=chart_path)
-
- async def fetch_charts(self):
- await asyncio.gather(
- *[
- self._helm(["repo", "add", repo.name, str(repo.url)])
- for repo in self.repositories
- ]
- )
- await self._helm(["repo", "update"])
-
- await asyncio.gather(*[self._fetch_chart(chart) for chart in self.charts])
-
-
-async def main():
- config = parse_yaml_file_as(Config, ".charts.yml")
- await config.fetch_charts()
-
-
-if __name__ == "__main__":
- asyncio.run(main())
diff --git a/tox.ini b/tox.ini
index a361b7d..6e55c35 100644
--- a/tox.ini
+++ b/tox.ini
@@ -29,17 +29,9 @@
[testenv:sync-charts]
skipsdist = True
deps =
- aiopath
- aiofiles!=24.1.0
- aioshutil
- asynctempfile
- GitPython
- platformdirs
- pydantic
- pydantic-yaml
- python-gerrit-api
+ chart-vendor
commands =
- python3 {toxinidir}/build/sync-charts.py
+ chart-vendor
[testenv:linters]
skipsdist = True
@@ -48,7 +40,7 @@
pre-commit
commands =
pre-commit run --all-files --show-diff-on-failure
- python3 {toxinidir}/build/sync-charts.py --check
+ chart-vendor --check
python3 {toxinidir}/build/lint-jobs.py
[testenv:py3]
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 7d76f03..dc7f780 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -2,6 +2,8 @@
name: atmosphere-linters
parent: tox-linters
pre-run: zuul.d/playbooks/linters/pre.yml
+ required-projects:
+ - name: vexxhost/chart-vendor
- job:
name: atmosphere-tox-py3