Merge pull request #75 from joaovitorsilvestre/improve_bootstrap_process

Improve bootstrap process + fydoc

.gitignore

@@ -5,5 +5,5 @@
 src/_compiled
 /example/
 **/erl_crash.dump
-libs/fy_doc/_env
+lib/fython/site
 bootstraped

Makefile

@@ -3,14 +3,17 @@ SHELL := /bin/bash
 # makefile location
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 # fython src folder
-SRC_DIR:="$(ROOT_DIR)/src"
+FYTHON_SRC_DIR:="$(ROOT_DIR)/lib/fython"
+
+# TODO these are tests of fython, we need to move it to fython folder inside lib
 TESTS_PATH:="$(ROOT_DIR)/tests"
 
 BOOTSTRAP_DOCKER_TAG:="fython:bootstrap"
 COMPILER_DOCKER_TAG:="fython:compiler"
 SHELL_DOCKER_TAG:="fython:shell"
 FYTEST_DOCKER_TAG:="fython:fytest"
 TESTS_DOCKER_TAG:="fython:tests"
+FYDOC_DOCKER_TAG:="fython:fydoc"
 
 .ONESHELL:
 bootstrap-with-docker:
@@ -20,7 +23,8 @@ bootstrap-with-docker:
 			--name fython_bootstrap \
 			-v $(ROOT_DIR)/bootstraped:/final_bootstrap \
 			-e FINAL_PATH='/final_bootstrap' \
-			$(BOOTSTRAP_DOCKER_TAG) && echo "Bootstrap finished. Result saved at '$(ROOT_DIR)/bootstraped'"
+			$(BOOTSTRAP_DOCKER_TAG) && echo "Bootstrap finished. Result saved at '$(ROOT_DIR)/bootstraped'" \
+		&& docker rm -f fython_bootstrap
 
 compile-project:
 	# USAGE:
@@ -33,12 +37,20 @@ run-tests:
 	$(MAKE) build-fytest \
 		&& DOCKER_BUILDKIT=1 docker run -e FOLDER=$(TESTS_PATH) -v $(TESTS_PATH):$(TESTS_PATH) $(FYTEST_DOCKER_TAG)
 
+gen-docs-fython:
+	# > make gen-docs
+	$(MAKE) build-fydoc \
+		&& DOCKER_BUILDKIT=1 docker run -e FOLDER=$(FYTHON_SRC_DIR) -v $(FYTHON_SRC_DIR):$(FYTHON_SRC_DIR) $(FYDOC_DOCKER_TAG)
+
 build-shell:
 	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(SHELL_DOCKER_TAG) --target shell .
 
 build-fytest:
 	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(FYTEST_DOCKER_TAG) --target fytest .
 
+build-fydoc:
+	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(FYDOC_DOCKER_TAG) --target fydoc .
+
 shell-current-src:
 	$(MAKE) build-shell \
 		&& DOCKER_BUILDKIT=1 docker run -it $(SHELL_DOCKER_TAG)

devtools/Dockerfile

@@ -2,7 +2,7 @@ FROM elixir:1.14.1 as base
 
 # Arguments you can modify
 ARG DESTINE_PATH="/compiled"
-ARG VERSION_TO_USE_AS_BOOTSTRAPER="v0.7.0"
+ARG VERSION_TO_USE_AS_BOOTSTRAPER="v0.8.0"
 
 ENV DESTINE_PATH=$DESTINE_PATH
 ENV PATH_FYTHON_TO_USE_AS_BOOTSTRAPER="/fython_$VERSION_TO_USE_AS_BOOTSTRAPER"
@@ -15,39 +15,39 @@ RUN wget https://github.com/joaovitorsilvestre/fython/releases/download/$VERSION
     && mkdir -p $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER \
     && tar -xf $OUTPUT_ZIP_PATH -C $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER
 
-COPY src src
+COPY lib/fython lib/fython
 COPY /devtools/bootstrap.sh /bootstrap.sh
-COPY /devtools/bootstrap_old.sh /bootstrap_old.sh
 COPY /devtools/fython /fython
 RUN chmod a+x /bootstrap.sh
-RUN chmod a+x /bootstrap_old.sh
 RUN chmod a+x /fython
 
 # Execute bootstrap
 RUN echo "Bootstraping using Fython $VERSION_TO_USE_AS_BOOTSTRAPER" \
-    && ./bootstrap_old.sh compile /src $DESTINE_PATH $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER $ELIXIR_BEAMS_PATH
+    && ./bootstrap.sh compile /lib/fython $DESTINE_PATH $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER $ELIXIR_BEAMS_PATH
 
 # Remove base used for bootstrap (to ensure that we are not using it anymore)
 RUN rm -rf $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER
 
 FROM base as bootstrap
-ENV FIRST_PATH="/test_compiled1"
-ENV SECOND_PATH="/test_compiled2"
 
-# Check if it can recompile itself (its a good way to test while we dont have proper testing)
-RUN ./bootstrap.sh compile /src $FIRST_PATH $DESTINE_PATH $ELIXIR_BEAMS_PATH
-RUN ./bootstrap.sh compile /src $SECOND_PATH $FIRST_PATH $ELIXIR_BEAMS_PATH
+ENV FIRST_BOOTSTRAP_DESTINE="/bootstrap1"
+ENV SECOND_BOOTSTRAP_DESTINE="/bootstrap2"
 
-# The final version bootstraped ready for release
+RUN /fython bootstrap /lib/fython $FIRST_BOOTSTRAP_DESTINE $DESTINE_PATH $ELIXIR_BEAMS_PATH
+
+# Check if it can recompile itself
+RUN /fython bootstrap /lib/fython $SECOND_BOOTSTRAP_DESTINE $FIRST_BOOTSTRAP_DESTINE $ELIXIR_BEAMS_PATH
+
+## The final version bootstraped ready for release
 ENV FINAL_PATH="/bootstraped"
-CMD ./bootstrap.sh compile /src $FINAL_PATH $SECOND_PATH $ELIXIR_BEAMS_PATH
+CMD /fython bootstrap /lib/fython $FINAL_PATH $SECOND_BOOTSTRAP_DESTINE $ELIXIR_BEAMS_PATH
 
 FROM base as shell
 ENV ADITIONAL_PATHS=""
-ENV SHELL_PATH="/libs/shell"
-ENV SHELL_PATH_COMPILED="/libs/shell/_compiled"
+ENV SHELL_PATH="/lib/shell"
+ENV SHELL_PATH_COMPILED="/lib/shell/_compiled"
 
-COPY libs/shell $SHELL_PATH
+COPY lib/shell $SHELL_PATH
 RUN /fython exec "Core.Code.compile_project('$SHELL_PATH')" $DESTINE_PATH
 
 CMD /fython exec "Shell.start()" $SHELL_PATH_COMPILED $DESTINE_PATH $ADITIONAL_PATHS
@@ -62,12 +62,40 @@ FROM base as fytest
 
 ENV FOLDER="MUST_BE_GIVEN"
 
-ENV FYTEST_PATH="/libs/fytest"
-ENV FYTEST_PATH_COMPILED="/libs/fytest/_compiled"
+ENV FYTEST_PATH="/lib/fytest"
+ENV FYTEST_PATH_COMPILED="/lib/fytest/_compiled"
 
-COPY libs/fytest $FYTEST_PATH
+COPY lib/fytest $FYTEST_PATH
 RUN /fython exec "Core.Code.compile_project('$FYTEST_PATH')" $DESTINE_PATH
 
 CMD echo "Compiling $FOLDER" \
     && /fython exec "Core.Code.compile_project('$FOLDER')" $DESTINE_PATH \
     && /fython exec "Fytest.run('$FOLDER')" $FOLDER/_compiled $FYTEST_PATH_COMPILED $DESTINE_PATH
+
+FROM base as fydoc
+
+# Install python to use for docs
+RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    apt update && \
+    apt install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt install -y python3.9 python3-pip
+
+ENV FOLDER="MUST_BE_GIVEN"
+
+ENV FYDOC_PATH="/lib/fydoc"
+COPY /lib/fydoc $FYDOC_PATH
+
+RUN pip3 install -r $FYDOC_PATH/mkdocs/requirements.txt
+
+RUN /fython exec "Core.Code.compile_project('$FYDOC_PATH')" $DESTINE_PATH
+
+CMD rm -rf $FOLDER/docs && \
+    /fython exec "Fydoc.run('$FOLDER')" $FYDOC_PATH/_compiled $DESTINE_PATH && \
+    cp $FYDOC_PATH/mkdocs/docs/extra.css $FOLDER/docs/extra.css && \
+    cp $FYDOC_PATH/mkdocs/docs/extra.js $FOLDER/docs/extra.js && \
+    cp $FYDOC_PATH/mkdocs/docs/index.md $FOLDER/docs/index.md && \
+    cp $FYDOC_PATH/mkdocs/mkdocs.yml $FOLDER/mkdocs.yml && \
+    cd $FOLDER && mkdocs build && rm -rf $FOLDER/mkdocs.yml && rm -rf $FOLDER/docs && \
+    echo "Documentation generated at $FOLDER/site"

devtools/fython

@@ -12,4 +12,42 @@ exec () {
   erl $ERLANG_PATHS_FORMATED -noshell -eval "'Fython.Core':eval_string(<<"'"'"$FYTHON_CODE"'"'">>)." -run init stop
 }
 
+bootstrap () {
+  FYTHON_CODE_PATH=$1
+  SECOND_COMPILED_FOLDER=$2
+  FYTHON_BEAMS_USE_TO_BOOTSTRAP=$3
+  ELIXIR_BEAMS_PATH=$4
+
+  [[ -z "$FYTHON_CODE_PATH" ]] && echo "Missing fython src code path" && exit 1
+  [[ -z "$FYTHON_BEAMS_USE_TO_BOOTSTRAP" ]] && echo "Missing fython's beams folder" && exit 1
+  [[ -z "$ELIXIR_BEAMS_PATH" ]] && echo "Missing elixir's beams folder" && exit 1
+
+  PREFIX=Bootstrap
+  FIRST_COMPILED_FOLDER=/test_compiled1
+
+  # 1º Compile with a module prefix to prevent modules overriding
+
+  rm -rf $FIRST_COMPILED_FOLDER/*
+  erl -pa $FYTHON_BEAMS_USE_TO_BOOTSTRAP \
+    -noshell \
+    -eval "'Fython.Core.Code':compile_project(<<"'"'"$FYTHON_CODE_PATH"'"'">>, <<"'"'"$FIRST_COMPILED_FOLDER"'"'">>, <<"'"'"$PREFIX"'"'">>)." \
+    -run init stop
+
+  cd $ELIXIR_BEAMS_PATH && cp -r . "$FIRST_COMPILED_FOLDER" && cd /
+
+  # 2º Bootstrap again using previous step, but without the prefix this time
+  # so we have the final bootstraped beams
+
+  rm -rf $SECOND_COMPILED_FOLDER/*
+  erl -pa $FIRST_COMPILED_FOLDER \
+    -noshell \
+    -eval "'Fython.$PREFIX.Core.Code':compile_project(<<"'"'"$FYTHON_CODE_PATH"'"'">>, <<"'"'"$SECOND_COMPILED_FOLDER"'"'">>, nil)." \
+    -run init stop
+
+  cd $ELIXIR_BEAMS_PATH && cp -r . "$SECOND_COMPILED_FOLDER" && cd /
+
+  # Creaning up disk
+  rm -rf $FIRST_COMPILED_FOLDER
+}
+
 $*

lib/fydoc/__init__.fy

@@ -0,0 +1,4 @@
+def run(project_path):
+    project_path
+        |> Fydoc.Scanner.get_functions_defs()
+        |> Fydoc.Generator.gererate_docs(project_path)

lib/fydoc/generator.fy

@@ -0,0 +1,50 @@
+def gererate_docs(module_n_functions, project_path):
+    module_n_functions
+        |> Elixir.Enum.map(lambda x: generate_doc(x, project_path))
+
+
+def generate_doc((module_name, functions), project_path):
+    sumary = functions
+        |> Elixir.Enum.with_index()
+        |> Elixir.Enum.map(lambda (func, index):
+            Elixir.Enum.join([
+                '<a href="#func', index, '" class="function-sumary-header">',
+                func,
+                '</a>'
+            ])
+        )
+
+    functions = functions
+        |> Elixir.Enum.with_index()
+        |> Elixir.Enum.map(lambda (func, index):
+            Elixir.Enum.join([
+                '<h3 class="function-header" id="func', index,'">',
+                func,
+                '</h3>'
+            ])
+        )
+
+    splited_module_name = module_name |> Elixir.String.split('.')
+
+    file_content = Elixir.Enum.join([
+        Elixir.Enum.join(["## ", module_name]),
+        "### Sumary",
+        *sumary,
+        "### Functions",
+        *functions
+    ], "\n\n")
+
+    result_file_name_with_path = generate_file_path(module_name, project_path)
+    Elixir.File.mkdir_p!(result_file_name_with_path |> Elixir.Path.dirname())
+
+    Elixir.File.write(result_file_name_with_path, file_content)
+
+
+def generate_file_path(module_name, project_path):
+    # returns docs/MyModule/ChildModule.md
+    Elixir.Enum.join([
+        project_path |> Elixir.String.replace_trailing("/", ""),
+        "/docs/modules/",
+        module_name |> Elixir.String.split('.') |> Elixir.Enum.join('/'),
+        ".md"
+    ])

lib/fydoc/mkdocs/docs/extra.css

@@ -0,0 +1,17 @@
+.function-header {
+    margin: 2em 0 1em;
+    padding: 0.5em 1em;
+    background: #f7f7f7;
+    border-left: 3px solid #6ecb58;
+    font-size: 1.2em !important;
+    font-weight: 700;
+    font-family: Inconsolata,Menlo,Courier,monospace;
+    position: relative;
+}
+
+.function-sumary-header {
+    color: rgb(51, 51, 51) !important;
+    text-decoration: none !important;
+    font-weight: 700;
+    font-size: 1.1em;
+}

lib/fydoc/mkdocs/docs/index.md

@@ -0,0 +1,3 @@
+# Home
+
+Documentation generated by fydoc

lib/fydoc/mkdocs/mkdocs.yml

@@ -0,0 +1,10 @@
+site_name: Fython
+theme:
+  name: gitbook
+extra_css: ["extra.css"]
+extra_javascript: ["extra.js"]
+#nav:
+#    - 'Home': 'index.md'
+#    - 'Modules':
+#        - 'Core':
+#            - Functions: 'modules/core/functions.md'

lib/fydoc/mkdocs/requirements.txt

@@ -0,0 +1,15 @@
+click==8.1.3
+ghp-import==2.1.0
+Jinja2==3.1.2
+Markdown==3.3.7
+MarkupSafe==2.1.1
+mergedeep==1.3.4
+mkdocs==1.4.2
+mkdocs-gitbook==0.0.1
+packaging==21.3
+pyparsing==3.0.9
+python-dateutil==2.8.2
+PyYAML==6.0
+pyyaml_env_tag==0.1
+six==1.16.0
+watchdog==2.1.9