-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #14 from openinfradev/render
new render
Showing
9 changed files
with
564 additions
and
367 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
set -ex | ||
|
||
VER=v3.0.0 | ||
IMG=harbor.taco-cat.xyz/tks/decapod-render:${VER} | ||
|
||
docker build . -f package/Dockerfile --network host --tag ${IMG} | ||
# docker build ./package --network host --tag $IMG | ||
sudo docker push $IMG |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,179 @@ | ||
from tokenize import String | ||
from typing import Dict, List, Optional, Any | ||
import sys | ||
import yaml | ||
import os | ||
import time | ||
import copy | ||
from .helm import Helm | ||
from .repo import Repo, RepoType | ||
from .parameter_aware_yamlmerge import yaml_override, traverse_leaf | ||
|
||
class HelmDeployManager: | ||
class Config: | ||
DEFAULT_CHECK_INTERVAL = 10 | ||
DEFAULT_KUBECONFIG = '~/.kube/config' | ||
|
||
def __init__(self, kubeconfig: str = Config.DEFAULT_KUBECONFIG, base: str = '', namespace: str = '', verbose: int = 0): | ||
self.kubeconfig = kubeconfig | ||
self.namespace = namespace | ||
self.verbose = verbose | ||
self.base: Dict[str, Helm] = self.__load_base(base) | ||
self.manifests = None #Dict[str, Helm] | ||
|
||
def get_repo(self, parsed: Dict[str, Any]) -> Repo: | ||
chart_spec = parsed['spec']['chart'] | ||
if 'type' in chart_spec: | ||
repo_type = RepoType.GIT if chart_spec['type'] == 'git' else RepoType.HELMREPO | ||
return Repo(repo_type, chart_spec['repository'], chart_spec['name'], chart_spec['version']) | ||
elif 'git' in chart_spec: | ||
return Repo(RepoType.GIT, chart_spec['git'], chart_spec['path'], chart_spec['ref']) | ||
elif 'repository' in chart_spec: | ||
return Repo(RepoType.HELMREPO, chart_spec['repository'], chart_spec['name'], chart_spec['version']) | ||
else: | ||
raise ValueError(f'Wrong repo {parsed}') | ||
|
||
def __load_base(self, base: str) -> Dict[str, Helm]: | ||
manifests: Dict[str, Helm] = {} | ||
if os.path.isdir(base): | ||
for item in os.listdir(base): | ||
item_path = os.path.join(base, item) | ||
if os.path.isdir(item_path) or item == 'resources.yaml': | ||
manifests.update(self.__load_base(item_path)) | ||
elif os.path.isfile(base): | ||
with open(base, 'r') as f: | ||
for parsed in yaml.safe_load_all(f): | ||
if parsed is None or 'spec' not in parsed: | ||
print(f'--- Warn: An invalid resource is given ---\n{parsed}\n--- Warn END ---') | ||
continue | ||
try: | ||
name = parsed['metadata']['name'] | ||
if name in manifests: | ||
manifests[name].override = yaml_override(manifests[name].override, parsed['spec']['values']) | ||
else: | ||
manifests[name] = Helm( | ||
self.get_repo(parsed), | ||
parsed['spec']['releaseName'], | ||
parsed['spec']['targetNamespace'], | ||
parsed['spec']['values'] | ||
) | ||
except (yaml.YAMLError, TypeError, KeyError) as exc: | ||
print(f'Error processing yaml: {exc}') | ||
else: | ||
print(f"Error: {base} is neither a file nor a directory") | ||
return manifests | ||
|
||
def template_yaml(self, gdir: str = 'cd') -> None: | ||
for chart, helm in self.manifests.items(): | ||
if self.verbose > 0: | ||
print(f'(DEBUG) Generate resource yamls from {chart}\n{helm}') | ||
helm.to_separated_resources(gdir, self.verbose) | ||
|
||
def install_and_check_done(self, install: List[str], config: Dict[str, Any], skip_exist: bool = False) -> None: | ||
check_interval = config.get('metadata', {}).get('checkInterval', self.Config.DEFAULT_CHECK_INTERVAL) | ||
pending = [chart for chart in install if not (skip_exist and manifests[chart].get_status(self.kubeconfig) == 'deployed')] | ||
|
||
for chart in pending: | ||
print(f'{self.manifests[chart].override}') | ||
self.manifests[chart].install(kubeconfig=self.kubeconfig, target_namespace=self.namespace, verbose=self.verbose) | ||
|
||
while pending: | ||
pending = [chart for chart in pending if self.manifests[chart].get_status(self.kubeconfig) != 'deployed'] | ||
if pending: | ||
print(f"\nWaiting for finish({check_interval} sec): {pending}") | ||
time.sleep(check_interval) | ||
|
||
def delete_and_check_done(self, delete: List[str], config: Dict[str, Any]) -> None: | ||
for chart in delete: | ||
if os.system(f'helm delete -n {self.namespace} {chart} --kubeconfig {self.kubeconfig}') != 0: | ||
print(f'FAILED!! {chart} ({self.namespace}) cannot be deleted due to the errors above.') | ||
sys.exit(1) | ||
|
||
|
||
def load_site(self, site: str, prohibits: List[str]): | ||
self.manifests: Dict[str, Helm] = {} | ||
with open(site, 'r') as f: | ||
for parsed in yaml.safe_load_all(f): | ||
self.preprocess_site(parsed) | ||
for chart in parsed['charts']: | ||
chart_name = chart.get('name') | ||
base_name = chart.get('base', chart_name) | ||
self.manifests[chart_name] = copy.deepcopy(self.base[base_name]) | ||
self.manifests[chart_name].name = chart_name | ||
|
||
if 'override' in chart and chart['override']: | ||
self.manifests[chart_name].override = yaml_override(self.manifests[chart_name].override, chart['override']) | ||
if self.verbose: | ||
print(f'from ====> {self.manifests[chart_name].override}') | ||
self.manifests[chart_name].override = traverse_leaf(self.manifests[chart_name].override, parsed.get('global',{}), prohibits) | ||
if self.verbose: | ||
print(f'to ====> {self.manifests[chart_name].override}') | ||
|
||
def load_manifest(self, fd: Any, local_repository: Optional[str] = None) -> Dict[str, Helm]: | ||
manifests: Dict[str, Helm] = {} | ||
for parsed in yaml.safe_load_all(fd): | ||
if 'spec' not in parsed: | ||
print(f'--- Warn: An invalid resource is given ---\n{parsed}\n--- Warn END ---') | ||
continue | ||
repo = self.get_repo() | ||
if local_repository: | ||
repo.repo = local_repository | ||
|
||
manifests[parsed['metadata']['name']] = Helm( | ||
repo, | ||
parsed['spec']['releaseName'], | ||
parsed['spec']['targetNamespace'], | ||
parsed['spec']['values'] | ||
) | ||
return manifests | ||
|
||
def preprocess_site(self, site: Dict[str, Any]) -> None: | ||
for chart in site['charts']: | ||
if 'override' in chart and chart['override'] is not None: | ||
overrides = {} | ||
for k, v in chart['override'].items(): | ||
v = self.__split_keys(k.split('.'), v) if '.' in k else {k: v} | ||
overrides = yaml_override(overrides, v) | ||
chart['override'] = overrides | ||
|
||
def __split_keys(self, sp: List[str], value: Any) -> Dict[str, Any]: | ||
return {sp[0]: self.__split_keys(sp[1:], value)} if len(sp) > 1 else {sp[0]: value} | ||
|
||
def check_chart_repo(self, fd: Any, target_repo: str, except_list: List[str] = []) -> Dict[str, str]: | ||
invalid_dic = {} | ||
for parsed in yaml.safe_load_all(fd): | ||
repo = parsed.get('spec', {}).get('chart', {}).get('repository') | ||
if repo and not repo.startswith(target_repo) and repo not in except_list: | ||
name = parsed.get('metadata', {}).get('name') | ||
invalid_dic[name] = repo | ||
if self.verbose >= 1: | ||
print(f'{name} is defined with wrong repository: {repo}') | ||
return invalid_dic | ||
|
||
def check_image_repo(self, fd: Any, target_repo: str, except_list: List[str] = []) -> Dict[str, List[str]]: | ||
helm_dic = self.load_manifest(fd) | ||
if self.verbose > 0: | ||
print('(DEBUG) Loaded manifest:', helm_dic) | ||
for key, value in helm_dic.items(): | ||
print(key, value) | ||
|
||
invalid_dic = {} | ||
for key, helm in helm_dic.items(): | ||
invalid_images = [image_url for image_url in helm.get_image_list(self.verbose) | ||
if not image_url.startswith(target_repo) and image_url not in except_list] | ||
if invalid_images: | ||
invalid_dic[key] = invalid_images | ||
|
||
return invalid_dic | ||
|
||
def __str__(self): | ||
sl=[f'(DEBUG) Loaded base manifest: {self.base}'] | ||
if self.verbose: | ||
for key, value in self.base.items(): | ||
sl.append(f'(DEBUG) key: {key}, {value}') | ||
|
||
if self.verbose and self.manifests: | ||
sl.append(f'(DEBUG) Loaded base manifest: {self.manifests}') | ||
for key, value in self.manifests.items(): | ||
sl.append(f'(DEBUG) key: {key}, {value}') | ||
return "\n".join(sl) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
from typing import Any, Dict, List | ||
|
||
VAR_START = '$(' | ||
VAR_END = ')' | ||
VAR_START_OFFSET = len(VAR_START) | ||
|
||
def substitute_variables(v: str, var_dictionary: Dict[str, Any], prohibits: List[str]) -> Any: | ||
# print(f'v:{v}, type:{type(v)},Dict:{var_dictionary}') | ||
if not var_dictionary or VAR_START not in v: | ||
return check_prohibits(v, prohibits) | ||
|
||
# print(f'substitute_variables: input={v} ', end=" ") | ||
|
||
try: | ||
if v.startswith(VAR_START) and v.endswith(VAR_END) and v.count(VAR_START) == 1 and v.count(VAR_END) == 1: | ||
v = var_dictionary[v[VAR_START_OFFSET:-1].strip()] | ||
else: | ||
while VAR_START in v: | ||
sp = v.find(VAR_START) | ||
ep = v.find(VAR_END, sp) | ||
if ep < 0: | ||
raise ValueError(f'The value "{v}" has wrong format. Fix it and try again.') | ||
var = v[sp + VAR_START_OFFSET:ep].strip() | ||
v = v.replace(f'{VAR_START}{var}{VAR_END}', str(var_dictionary[var])) | ||
except KeyError: | ||
print(f''' | ||
################################################################################## | ||
- The "{v}" have any undefined variables. Fix it and try again. | ||
- Dictionary : {var_dictionary} | ||
################################################################################## | ||
''') | ||
raise ValueError(f'The "{v}" have any undefined variables. Fix it and try again.') | ||
|
||
return check_prohibits(v, prohibits) | ||
|
||
def check_prohibits(v: Any, prohibits: List[str]) -> Any: | ||
# 금지어 포함여부 체크 및 에러발생 | ||
if prohibits and isinstance(v, str) and len(v) > 2 and any(v in w for w in prohibits): | ||
print(f''' | ||
################################################################################## | ||
- You cannot use the string "{v}" as a value | ||
- Prohibit List : {prohibits} | ||
################################################################################## | ||
''') | ||
raise ValueError(f'You cannot use the string "{v}" as a value') | ||
return v | ||
|
||
def traverse_leaf(values: Any, var_dictionary: Dict[str, Any], prohibits: List[str]) -> Any: | ||
if isinstance(values, dict): | ||
return {k: traverse_leaf(v, var_dictionary, prohibits) for k, v in values.items()} | ||
elif isinstance(values, list): | ||
return [traverse_leaf(v, var_dictionary, prohibits) for v in values] | ||
elif isinstance(values, str): | ||
return substitute_variables(values, var_dictionary, prohibits) | ||
return values | ||
|
||
def yaml_override(target: Dict[str, Any], v: Any) -> Dict[str, Any]: | ||
if not target: | ||
return v | ||
if isinstance(v, dict): | ||
for k, val in v.items(): | ||
target[k] = yaml_override(target.get(k, {}), val) | ||
return target | ||
return v |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,63 +1,42 @@ | ||
from enum import Enum, unique | ||
import sys, yaml, os, time, getopt | ||
from typing import Optional | ||
|
||
|
||
@unique | ||
class RepoType(Enum): | ||
HELMREPO=1 | ||
GIT=2 | ||
LOCAL=3 | ||
HELMREPO = 1 | ||
GIT = 2 | ||
LOCAL = 3 | ||
|
||
class Repo: | ||
# Will Make When it's needed | ||
def __init__(self): | ||
self.list=[] | ||
|
||
def __init__(self, repotype, repo, chartOrPath, versionOrReference): | ||
def __init__(self, repotype: RepoType, repo: str, chart_or_path: str, version_or_reference: str): | ||
self.repotype = repotype | ||
self.repo = repo | ||
self.chartOrPath = chartOrPath | ||
self.versionOrReference = versionOrReference | ||
|
||
def toString(self): | ||
return '[REPO: {}, {}, {}, {}]'.format( self.repotype, | ||
self.repo, | ||
self.chartOrPath, | ||
self.versionOrReference ) | ||
|
||
def version(self): | ||
if self.repotype == RepoType.HELMREPO: | ||
return self.versionOrReference | ||
else: | ||
return None | ||
|
||
def reference(self): | ||
if self.repotype == RepoType.GIT: | ||
return self.versionOrReference | ||
else: | ||
return None | ||
|
||
def chart(self): | ||
if self.repotype == RepoType.HELMREPO: | ||
return self.chartOrPath | ||
else: | ||
return None | ||
def path(self): | ||
if self.repotype == RepoType.GIT: | ||
return self.chartOrPath | ||
else: | ||
return None | ||
self.chart_or_path = chart_or_path | ||
self.version_or_reference = version_or_reference | ||
|
||
def __str__(self) -> str: | ||
return f'[REPO: {self.repotype} {self.repo}, {self.chart_or_path}, {self.version_or_reference}]' | ||
|
||
def version(self) -> Optional[str]: | ||
return self.version_or_reference if self.repotype == RepoType.HELMREPO else None | ||
|
||
def reference(self) -> Optional[str]: | ||
return self.version_or_reference if self.repotype == RepoType.GIT else None | ||
|
||
def chart(self) -> Optional[str]: | ||
return self.chart_or_path if self.repotype == RepoType.HELMREPO else None | ||
|
||
def path(self) -> Optional[str]: | ||
return self.chart_or_path if self.repotype == RepoType.GIT else None | ||
|
||
def repository(self): | ||
def repository(self) -> str: | ||
return self.repo | ||
|
||
def getUrl(self): | ||
if self.repotype == RepoType.GIT and self.repo.startswith('git@'): | ||
if self.repo.endswith('.git'): | ||
return self.repo.replace(':','/').replace('git@','https://') | ||
else: | ||
return self.repo.replace(':','/').replace('git@','https://')+'.git' | ||
|
||
def get_url(self) -> Optional[str]: | ||
if self.repotype == RepoType.GIT: | ||
if self.repo.startswith('git@'): | ||
url = self.repo.replace(':', '/').replace('git@', 'https://') | ||
return url if url.endswith('.git') else f"{url}.git" | ||
return self.repo | ||
else: | ||
return None | ||
return None |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
#!/usr/local/bin/python3 | ||
|
||
import sys | ||
import argparse | ||
from typing import Dict, Any | ||
from applib.helmdeploy import HelmDeployManager | ||
|
||
class AppConfig: | ||
DEFAULT_WORKFLOW = 'default' | ||
DEFAULT_KUBECONFIG = '~/.kube/config' | ||
DEFAULT_OUTPUT_DIR = 'cd' | ||
DEFAULT_BASE = './base/ai-base.yaml' | ||
|
||
|
||
def parse_arguments() -> argparse.Namespace: | ||
parser = argparse.ArgumentParser(description="Process app manifest and generate YAML files") | ||
parser.add_argument('-o', '--override', required=True, help="The manifest file with override values") | ||
parser.add_argument('-b', '--base', default=AppConfig.DEFAULT_BASE, help="The base manifest file") | ||
parser.add_argument('-v', '--verbose', action='count', default=0, help="Increase verbosity") | ||
parser.add_argument('-k', '--kubeconfig', default=AppConfig.DEFAULT_KUBECONFIG, | ||
help="Kubeconfig file to access your k8s cluster") | ||
parser.add_argument('-n', '--namespace', help="Define namespace to install regardless of the manifest") | ||
parser.add_argument('--output', default=AppConfig.DEFAULT_OUTPUT_DIR, help="Output directory for YAML files") | ||
return parser.parse_args() | ||
|
||
|
||
def print_debug_info(manifests: Dict[str, Any]) -> None: | ||
print('(DEBUG) loaded manifest:', manifests) | ||
for key, value in manifests.items(): | ||
print(key, value) | ||
|
||
|
||
def main() -> None: | ||
args = parse_arguments() | ||
|
||
helm_manager = HelmDeployManager(kubeconfig=args.kubeconfig, base=args.base, namespace=args.namespace, verbose=args.verbose) | ||
helm_manager.load_site(args.override, ['CHANGEME','FIXME','TODO']) | ||
|
||
helm_manager.template_yaml(args.output) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
FROM python:3.12.4-alpine | ||
|
||
LABEL AUTHOR sungil(usnexp@gmail.com) | ||
|
||
ENV HELM_VER v3.9.0 | ||
ENV GH_VER 2.11.3 | ||
|
||
RUN apk update && apk add curl | ||
RUN curl -sL -o helm.tar.gz https://get.helm.sh/helm-${HELM_VER}-linux-amd64.tar.gz && \ | ||
tar xf helm.tar.gz && \ | ||
mv linux-amd64/helm /usr/local/bin/helm | ||
|
||
RUN curl -sL -o gh.tar.gz https://github.com/cli/cli/releases/download/v${GH_VER}/gh_${GH_VER}_linux_amd64.tar.gz && \ | ||
tar xf gh.tar.gz && \ | ||
mv gh_2.11.3_linux_amd64/bin/gh /usr/local/bin/gh | ||
|
||
COPY helm2yaml /helm2yaml | ||
|
||
COPY requirements.txt requirements.txt | ||
RUN pip3 install -r requirements.txt | ||
|
||
ENTRYPOINT [ "/helm2yaml/template" ] | ||
# sktcloud/decapod-render:v3.0.0 (23.8.14) | ||
# - support oci protocol for a helm repository | ||
# - generate CRDs using helm cli | ||
# Build CLI: docker build --network host -t siim/helm2yaml:v3.0.0 . |