Skip to content

Commit

Permalink
Merge pull request #14 from openinfradev/render
Browse files Browse the repository at this point in the history
new render
intelliguy authored Sep 10, 2024
2 parents dbf6f03 + d2b0f9c commit aee8863
Showing 9 changed files with 564 additions and 367 deletions.
41 changes: 0 additions & 41 deletions Dockerfile

This file was deleted.

8 changes: 8 additions & 0 deletions docker-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
set -ex

VER=v3.0.0
IMG=harbor.taco-cat.xyz/tks/decapod-render:${VER}

docker build . -f package/Dockerfile --network host --tag ${IMG}
# docker build ./package --network host --tag $IMG
sudo docker push $IMG
487 changes: 213 additions & 274 deletions helm2yaml/applib/helm.py

Large diffs are not rendered by default.

179 changes: 179 additions & 0 deletions helm2yaml/applib/helmdeploy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
from tokenize import String
from typing import Dict, List, Optional, Any
import sys
import yaml
import os
import time
import copy
from .helm import Helm
from .repo import Repo, RepoType
from .parameter_aware_yamlmerge import yaml_override, traverse_leaf

class HelmDeployManager:
class Config:
DEFAULT_CHECK_INTERVAL = 10
DEFAULT_KUBECONFIG = '~/.kube/config'

def __init__(self, kubeconfig: str = Config.DEFAULT_KUBECONFIG, base: str = '', namespace: str = '', verbose: int = 0):
self.kubeconfig = kubeconfig
self.namespace = namespace
self.verbose = verbose
self.base: Dict[str, Helm] = self.__load_base(base)
self.manifests = None #Dict[str, Helm]

def get_repo(self, parsed: Dict[str, Any]) -> Repo:
chart_spec = parsed['spec']['chart']
if 'type' in chart_spec:
repo_type = RepoType.GIT if chart_spec['type'] == 'git' else RepoType.HELMREPO
return Repo(repo_type, chart_spec['repository'], chart_spec['name'], chart_spec['version'])
elif 'git' in chart_spec:
return Repo(RepoType.GIT, chart_spec['git'], chart_spec['path'], chart_spec['ref'])
elif 'repository' in chart_spec:
return Repo(RepoType.HELMREPO, chart_spec['repository'], chart_spec['name'], chart_spec['version'])
else:
raise ValueError(f'Wrong repo {parsed}')

def __load_base(self, base: str) -> Dict[str, Helm]:
manifests: Dict[str, Helm] = {}
if os.path.isdir(base):
for item in os.listdir(base):
item_path = os.path.join(base, item)
if os.path.isdir(item_path) or item == 'resources.yaml':
manifests.update(self.__load_base(item_path))
elif os.path.isfile(base):
with open(base, 'r') as f:
for parsed in yaml.safe_load_all(f):
if parsed is None or 'spec' not in parsed:
print(f'--- Warn: An invalid resource is given ---\n{parsed}\n--- Warn END ---')
continue
try:
name = parsed['metadata']['name']
if name in manifests:
manifests[name].override = yaml_override(manifests[name].override, parsed['spec']['values'])
else:
manifests[name] = Helm(
self.get_repo(parsed),
parsed['spec']['releaseName'],
parsed['spec']['targetNamespace'],
parsed['spec']['values']
)
except (yaml.YAMLError, TypeError, KeyError) as exc:
print(f'Error processing yaml: {exc}')
else:
print(f"Error: {base} is neither a file nor a directory")
return manifests

def template_yaml(self, gdir: str = 'cd') -> None:
for chart, helm in self.manifests.items():
if self.verbose > 0:
print(f'(DEBUG) Generate resource yamls from {chart}\n{helm}')
helm.to_separated_resources(gdir, self.verbose)

def install_and_check_done(self, install: List[str], config: Dict[str, Any], skip_exist: bool = False) -> None:
check_interval = config.get('metadata', {}).get('checkInterval', self.Config.DEFAULT_CHECK_INTERVAL)
pending = [chart for chart in install if not (skip_exist and manifests[chart].get_status(self.kubeconfig) == 'deployed')]

for chart in pending:
print(f'{self.manifests[chart].override}')
self.manifests[chart].install(kubeconfig=self.kubeconfig, target_namespace=self.namespace, verbose=self.verbose)

while pending:
pending = [chart for chart in pending if self.manifests[chart].get_status(self.kubeconfig) != 'deployed']
if pending:
print(f"\nWaiting for finish({check_interval} sec): {pending}")
time.sleep(check_interval)

def delete_and_check_done(self, delete: List[str], config: Dict[str, Any]) -> None:
for chart in delete:
if os.system(f'helm delete -n {self.namespace} {chart} --kubeconfig {self.kubeconfig}') != 0:
print(f'FAILED!! {chart} ({self.namespace}) cannot be deleted due to the errors above.')
sys.exit(1)


def load_site(self, site: str, prohibits: List[str]):
self.manifests: Dict[str, Helm] = {}
with open(site, 'r') as f:
for parsed in yaml.safe_load_all(f):
self.preprocess_site(parsed)
for chart in parsed['charts']:
chart_name = chart.get('name')
base_name = chart.get('base', chart_name)
self.manifests[chart_name] = copy.deepcopy(self.base[base_name])
self.manifests[chart_name].name = chart_name

if 'override' in chart and chart['override']:
self.manifests[chart_name].override = yaml_override(self.manifests[chart_name].override, chart['override'])
if self.verbose:
print(f'from ====> {self.manifests[chart_name].override}')
self.manifests[chart_name].override = traverse_leaf(self.manifests[chart_name].override, parsed.get('global',{}), prohibits)
if self.verbose:
print(f'to ====> {self.manifests[chart_name].override}')

def load_manifest(self, fd: Any, local_repository: Optional[str] = None) -> Dict[str, Helm]:
manifests: Dict[str, Helm] = {}
for parsed in yaml.safe_load_all(fd):
if 'spec' not in parsed:
print(f'--- Warn: An invalid resource is given ---\n{parsed}\n--- Warn END ---')
continue
repo = self.get_repo()
if local_repository:
repo.repo = local_repository

manifests[parsed['metadata']['name']] = Helm(
repo,
parsed['spec']['releaseName'],
parsed['spec']['targetNamespace'],
parsed['spec']['values']
)
return manifests

def preprocess_site(self, site: Dict[str, Any]) -> None:
for chart in site['charts']:
if 'override' in chart and chart['override'] is not None:
overrides = {}
for k, v in chart['override'].items():
v = self.__split_keys(k.split('.'), v) if '.' in k else {k: v}
overrides = yaml_override(overrides, v)
chart['override'] = overrides

def __split_keys(self, sp: List[str], value: Any) -> Dict[str, Any]:
return {sp[0]: self.__split_keys(sp[1:], value)} if len(sp) > 1 else {sp[0]: value}

def check_chart_repo(self, fd: Any, target_repo: str, except_list: List[str] = []) -> Dict[str, str]:
invalid_dic = {}
for parsed in yaml.safe_load_all(fd):
repo = parsed.get('spec', {}).get('chart', {}).get('repository')
if repo and not repo.startswith(target_repo) and repo not in except_list:
name = parsed.get('metadata', {}).get('name')
invalid_dic[name] = repo
if self.verbose >= 1:
print(f'{name} is defined with wrong repository: {repo}')
return invalid_dic

def check_image_repo(self, fd: Any, target_repo: str, except_list: List[str] = []) -> Dict[str, List[str]]:
helm_dic = self.load_manifest(fd)
if self.verbose > 0:
print('(DEBUG) Loaded manifest:', helm_dic)
for key, value in helm_dic.items():
print(key, value)

invalid_dic = {}
for key, helm in helm_dic.items():
invalid_images = [image_url for image_url in helm.get_image_list(self.verbose)
if not image_url.startswith(target_repo) and image_url not in except_list]
if invalid_images:
invalid_dic[key] = invalid_images

return invalid_dic

def __str__(self):
sl=[f'(DEBUG) Loaded base manifest: {self.base}']
if self.verbose:
for key, value in self.base.items():
sl.append(f'(DEBUG) key: {key}, {value}')

if self.verbose and self.manifests:
sl.append(f'(DEBUG) Loaded base manifest: {self.manifests}')
for key, value in self.manifests.items():
sl.append(f'(DEBUG) key: {key}, {value}')
return "\n".join(sl)
64 changes: 64 additions & 0 deletions helm2yaml/applib/parameter_aware_yamlmerge.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
from typing import Any, Dict, List

VAR_START = '$('
VAR_END = ')'
VAR_START_OFFSET = len(VAR_START)

def substitute_variables(v: str, var_dictionary: Dict[str, Any], prohibits: List[str]) -> Any:
# print(f'v:{v}, type:{type(v)},Dict:{var_dictionary}')
if not var_dictionary or VAR_START not in v:
return check_prohibits(v, prohibits)

# print(f'substitute_variables: input={v} ', end=" ")

try:
if v.startswith(VAR_START) and v.endswith(VAR_END) and v.count(VAR_START) == 1 and v.count(VAR_END) == 1:
v = var_dictionary[v[VAR_START_OFFSET:-1].strip()]
else:
while VAR_START in v:
sp = v.find(VAR_START)
ep = v.find(VAR_END, sp)
if ep < 0:
raise ValueError(f'The value "{v}" has wrong format. Fix it and try again.')
var = v[sp + VAR_START_OFFSET:ep].strip()
v = v.replace(f'{VAR_START}{var}{VAR_END}', str(var_dictionary[var]))
except KeyError:
print(f'''
##################################################################################
- The "{v}" have any undefined variables. Fix it and try again.
- Dictionary : {var_dictionary}
##################################################################################
''')
raise ValueError(f'The "{v}" have any undefined variables. Fix it and try again.')

return check_prohibits(v, prohibits)

def check_prohibits(v: Any, prohibits: List[str]) -> Any:
# 금지어 포함여부 체크 및 에러발생
if prohibits and isinstance(v, str) and len(v) > 2 and any(v in w for w in prohibits):
print(f'''
##################################################################################
- You cannot use the string "{v}" as a value
- Prohibit List : {prohibits}
##################################################################################
''')
raise ValueError(f'You cannot use the string "{v}" as a value')
return v

def traverse_leaf(values: Any, var_dictionary: Dict[str, Any], prohibits: List[str]) -> Any:
if isinstance(values, dict):
return {k: traverse_leaf(v, var_dictionary, prohibits) for k, v in values.items()}
elif isinstance(values, list):
return [traverse_leaf(v, var_dictionary, prohibits) for v in values]
elif isinstance(values, str):
return substitute_variables(values, var_dictionary, prohibits)
return values

def yaml_override(target: Dict[str, Any], v: Any) -> Dict[str, Any]:
if not target:
return v
if isinstance(v, dict):
for k, val in v.items():
target[k] = yaml_override(target.get(k, {}), val)
return target
return v
79 changes: 29 additions & 50 deletions helm2yaml/applib/repo.py
Original file line number Diff line number Diff line change
@@ -1,63 +1,42 @@
from enum import Enum, unique
import sys, yaml, os, time, getopt
from typing import Optional


@unique
class RepoType(Enum):
HELMREPO=1
GIT=2
LOCAL=3
HELMREPO = 1
GIT = 2
LOCAL = 3

class Repo:
# Will Make When it's needed
def __init__(self):
self.list=[]

def __init__(self, repotype, repo, chartOrPath, versionOrReference):
def __init__(self, repotype: RepoType, repo: str, chart_or_path: str, version_or_reference: str):
self.repotype = repotype
self.repo = repo
self.chartOrPath = chartOrPath
self.versionOrReference = versionOrReference

def toString(self):
return '[REPO: {}, {}, {}, {}]'.format( self.repotype,
self.repo,
self.chartOrPath,
self.versionOrReference )

def version(self):
if self.repotype == RepoType.HELMREPO:
return self.versionOrReference
else:
return None

def reference(self):
if self.repotype == RepoType.GIT:
return self.versionOrReference
else:
return None

def chart(self):
if self.repotype == RepoType.HELMREPO:
return self.chartOrPath
else:
return None
def path(self):
if self.repotype == RepoType.GIT:
return self.chartOrPath
else:
return None
self.chart_or_path = chart_or_path
self.version_or_reference = version_or_reference

def __str__(self) -> str:
return f'[REPO: {self.repotype} {self.repo}, {self.chart_or_path}, {self.version_or_reference}]'

def version(self) -> Optional[str]:
return self.version_or_reference if self.repotype == RepoType.HELMREPO else None

def reference(self) -> Optional[str]:
return self.version_or_reference if self.repotype == RepoType.GIT else None

def chart(self) -> Optional[str]:
return self.chart_or_path if self.repotype == RepoType.HELMREPO else None

def path(self) -> Optional[str]:
return self.chart_or_path if self.repotype == RepoType.GIT else None

def repository(self):
def repository(self) -> str:
return self.repo

def getUrl(self):
if self.repotype == RepoType.GIT and self.repo.startswith('git@'):
if self.repo.endswith('.git'):
return self.repo.replace(':','/').replace('git@','https://')
else:
return self.repo.replace(':','/').replace('git@','https://')+'.git'

def get_url(self) -> Optional[str]:
if self.repotype == RepoType.GIT:
if self.repo.startswith('git@'):
url = self.repo.replace(':', '/').replace('git@', 'https://')
return url if url.endswith('.git') else f"{url}.git"
return self.repo
else:
return None
return None
4 changes: 2 additions & 2 deletions helm2yaml/check_repo.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/usr/local/bin/python3
from applib.helm import Helm
from applib.repo import Repo, RepoType
import sys, os, time, getopt, yaml
from common import *
import sys, getopt
from monstar.installer.applib.helmdeploy import *

TEMPDIR="tmp"

43 changes: 43 additions & 0 deletions helm2yaml/template
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/usr/local/bin/python3

import sys
import argparse
from typing import Dict, Any
from applib.helmdeploy import HelmDeployManager

class AppConfig:
DEFAULT_WORKFLOW = 'default'
DEFAULT_KUBECONFIG = '~/.kube/config'
DEFAULT_OUTPUT_DIR = 'cd'
DEFAULT_BASE = './base/ai-base.yaml'


def parse_arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Process app manifest and generate YAML files")
parser.add_argument('-o', '--override', required=True, help="The manifest file with override values")
parser.add_argument('-b', '--base', default=AppConfig.DEFAULT_BASE, help="The base manifest file")
parser.add_argument('-v', '--verbose', action='count', default=0, help="Increase verbosity")
parser.add_argument('-k', '--kubeconfig', default=AppConfig.DEFAULT_KUBECONFIG,
help="Kubeconfig file to access your k8s cluster")
parser.add_argument('-n', '--namespace', help="Define namespace to install regardless of the manifest")
parser.add_argument('--output', default=AppConfig.DEFAULT_OUTPUT_DIR, help="Output directory for YAML files")
return parser.parse_args()


def print_debug_info(manifests: Dict[str, Any]) -> None:
print('(DEBUG) loaded manifest:', manifests)
for key, value in manifests.items():
print(key, value)


def main() -> None:
args = parse_arguments()

helm_manager = HelmDeployManager(kubeconfig=args.kubeconfig, base=args.base, namespace=args.namespace, verbose=args.verbose)
helm_manager.load_site(args.override, ['CHANGEME','FIXME','TODO'])

helm_manager.template_yaml(args.output)


if __name__ == "__main__":
main()
26 changes: 26 additions & 0 deletions package/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
FROM python:3.12.4-alpine

LABEL AUTHOR sungil(usnexp@gmail.com)

ENV HELM_VER v3.9.0
ENV GH_VER 2.11.3

RUN apk update && apk add curl
RUN curl -sL -o helm.tar.gz https://get.helm.sh/helm-${HELM_VER}-linux-amd64.tar.gz && \
tar xf helm.tar.gz && \
mv linux-amd64/helm /usr/local/bin/helm

RUN curl -sL -o gh.tar.gz https://github.com/cli/cli/releases/download/v${GH_VER}/gh_${GH_VER}_linux_amd64.tar.gz && \
tar xf gh.tar.gz && \
mv gh_2.11.3_linux_amd64/bin/gh /usr/local/bin/gh

COPY helm2yaml /helm2yaml

COPY requirements.txt requirements.txt
RUN pip3 install -r requirements.txt

ENTRYPOINT [ "/helm2yaml/template" ]
# sktcloud/decapod-render:v3.0.0 (23.8.14)
# - support oci protocol for a helm repository
# - generate CRDs using helm cli
# Build CLI: docker build --network host -t siim/helm2yaml:v3.0.0 .

0 comments on commit aee8863

Please sign in to comment.