Merge pull request #65 from joaovitorsilvestre/arrumando_a_casa

Arrumando a casa

.dockerignore

@@ -1,4 +1,5 @@
 **/_compiled/
 /bin/
 /Makefile
-**/erl_crash.dump
+**/erl_crash.dump
+bootstraped

.gitignore

@@ -5,4 +5,5 @@
 src/_compiled
 /example/
 **/erl_crash.dump
-fy_doc/_env
+libs/fy_doc/_env
+bootstraped

Makefile

@@ -4,28 +4,52 @@ SHELL := /bin/bash
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 # fython src folder
 SRC_DIR:="$(ROOT_DIR)/src"
+TESTS_PATH:="$(ROOT_DIR)/tests"
 
-bootstrap-with-docker:
-	@DOCKER_TAG="fython:bootstrap"
-	docker build -f devtools/Dockerfile -t $$DOCKER_TAG --target base .
+BOOTSTRAP_DOCKER_TAG:="fython:bootstrap"
+COMPILER_DOCKER_TAG:="fython:compiler"
+SHELL_DOCKER_TAG:="fython:shell"
+FYTEST_DOCKER_TAG:="fython:fytest"
+TESTS_DOCKER_TAG:="fython:tests"
 
 .ONESHELL:
+bootstrap-with-docker:
+	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(BOOTSTRAP_DOCKER_TAG) --target bootstrap . \
+		&& docker rm -f fython_bootstrap || true \
+		&& docker run \
+			--name fython_bootstrap \
+			-v $(ROOT_DIR)/bootstraped:/final_bootstrap \
+			-e FINAL_PATH='/final_bootstrap' \
+			$(BOOTSTRAP_DOCKER_TAG) && echo "Bootstrap finished. Result saved at '$(ROOT_DIR)/bootstraped'"
+
 compile-project:
 	# USAGE:
 	# > make compile-project path=/home/joao/fython/example
-	@DOCKER_TAG="fython:compiler"
-	docker build -f devtools/Dockerfile -t $$DOCKER_TAG --target compiler .
-	docker run --env PROJET_FOLDER=/project -v $(path):/project $$DOCKER_TAG
+	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(COMPILER_DOCKER_TAG) --target compiler . \
+		&& DOCKER_BUILDKIT=1 docker run --env PROJET_FOLDER=/project$(path) -v $(path):/project$(path) $(COMPILER_DOCKER_TAG)
 
-.ONESHELL:
 run-tests:
-	@DOCKER_TAG="fython:tests"
-	docker build -f devtools/Dockerfile -t $$DOCKER_TAG --target tests .
-	docker run $$DOCKER_TAG
+	# > make run-tests
+	$(MAKE) build-fytest \
+		&& DOCKER_BUILDKIT=1 docker run -e FOLDER=$(TESTS_PATH) -v $(TESTS_PATH):$(TESTS_PATH) $(FYTEST_DOCKER_TAG)
 
+build-shell:
+	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(SHELL_DOCKER_TAG) --target shell .
+
+build-fytest:
+	DOCKER_BUILDKIT=1 docker build -f devtools/Dockerfile -t $(FYTEST_DOCKER_TAG) --target fytest .
 
-.ONESHELL:
 shell-current-src:
-	@DOCKER_TAG="fython:shell"
-	docker build -f devtools/Dockerfile -t $$DOCKER_TAG --target shell .
-	docker run -it $$DOCKER_TAG
+	$(MAKE) build-shell \
+		&& DOCKER_BUILDKIT=1 docker run -it $(SHELL_DOCKER_TAG)
+
+project-shell:
+	# > project-shell path=/home/joao/fython/example
+	$(MAKE) compile-project path=$(path) \
+		&& $(MAKE) build-shell\
+		&& DOCKER_BUILDKIT=1 docker run -it --env ADITIONAL_PATHS=/project$(path)/_compiled -v $(path):/project$(path) $(SHELL_DOCKER_TAG)
+
+project-bash:
+	# > project-bash path=/home/joao/fython/example
+	$(MAKE) compile-project path=$(path) \
+		&& DOCKER_BUILDKIT=1 docker run -it --env ADITIONAL_PATHS=/project$(path)/_compiled -v $(path):/project$(path) $(SHELL_DOCKER_TAG) bash

devtools/Dockerfile

@@ -17,35 +17,57 @@ RUN wget https://github.com/joaovitorsilvestre/fython/releases/download/$VERSION
 
 COPY src src
 COPY /devtools/bootstrap.sh /bootstrap.sh
+COPY /devtools/bootstrap_old.sh /bootstrap_old.sh
 COPY /devtools/fython /fython
 RUN chmod a+x /bootstrap.sh
+RUN chmod a+x /bootstrap_old.sh
 RUN chmod a+x /fython
 
 # Execute bootstrap
 RUN echo "Bootstraping using Fython $VERSION_TO_USE_AS_BOOTSTRAPER" \
-    && ./bootstrap.sh compile /src $DESTINE_PATH $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER $ELIXIR_BEAMS_PATH
+    && ./bootstrap_old.sh compile /src $DESTINE_PATH $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER $ELIXIR_BEAMS_PATH
 
 # Remove base used for bootstrap (to ensure that we are not using it anymore)
 RUN rm -rf $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER
 
+FROM base as bootstrap
+ENV FIRST_PATH="/test_compiled1"
+ENV SECOND_PATH="/test_compiled2"
+
 # Check if it can recompile itself (its a good way to test while we dont have proper testing)
-#RUN ./bootstrap.sh compile /src /test_compiled1 $DESTINE_PATH $ELIXIR_BEAMS_PATH
-#RUN ./bootstrap.sh compile /src /test_compiled2 /test_compiled1
+RUN ./bootstrap.sh compile /src $FIRST_PATH $DESTINE_PATH $ELIXIR_BEAMS_PATH
+RUN ./bootstrap.sh compile /src $SECOND_PATH $FIRST_PATH $ELIXIR_BEAMS_PATH
 
-FROM base as elixir_shell
-# TODO
-# CMD erl -pa $IEX_BEAMS_PATH -noshell -user Elixir.IEx.CLI +iex
+# The final version bootstraped ready for release
+ENV FINAL_PATH="/bootstraped"
+CMD ./bootstrap.sh compile /src $FINAL_PATH $SECOND_PATH $ELIXIR_BEAMS_PATH
 
 FROM base as shell
-CMD /fython exec "Shell.start()" $DESTINE_PATH
+ENV ADITIONAL_PATHS=""
+ENV SHELL_PATH="/libs/shell"
+ENV SHELL_PATH_COMPILED="/libs/shell/_compiled"
+
+COPY libs/shell $SHELL_PATH
+RUN /fython exec "Core.Code.compile_project('$SHELL_PATH')" $DESTINE_PATH
+
+CMD /fython exec "Shell.start()" $SHELL_PATH_COMPILED $DESTINE_PATH $ADITIONAL_PATHS
 
 FROM base as compiler
 ENV PROJET_FOLDER="/project"
 CMD echo "Compiling project: $PROJET_FOLDER" \
+    && rm -rf $PROJET_FOLDER/_compiled \
     && /fython exec "Core.Code.compile_project('$PROJET_FOLDER')" $DESTINE_PATH
 
-FROM base as tests
-COPY tests tests
+FROM base as fytest
+
+ENV FOLDER="MUST_BE_GIVEN"
+
+ENV FYTEST_PATH="/libs/fytest"
+ENV FYTEST_PATH_COMPILED="/libs/fytest/_compiled"
+
+COPY libs/fytest $FYTEST_PATH
+RUN /fython exec "Core.Code.compile_project('$FYTEST_PATH')" $DESTINE_PATH
 
-CMD /fython exec "Core.Code.compile_project('/tests')" $DESTINE_PATH \
-    && /fython exec "Math_tests.run_tests()" $DESTINE_PATH /tests/_compiled
+CMD echo "Compiling $FOLDER" \
+    && /fython exec "Core.Code.compile_project('$FOLDER')" $DESTINE_PATH \
+    && /fython exec "Fytest.run('$FOLDER')" $FOLDER/_compiled $FYTEST_PATH_COMPILED $DESTINE_PATH

devtools/bootstrap.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e # stop if any error
+set -eo pipefail # stop if any error
 
 compile () {
   # We use the _COMP sufix to avoid conflict with external variables
@@ -8,19 +8,49 @@ compile () {
   DESTINE_PATH_COMP=$2
   PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP=$3
   ELIXIR_BEAMS_PATH_COMP=$4
+  COMPILE_IN_PARALEL=true
+
+  [[ -z "$SRC_DIR_COMP" ]] && echo "Missing src dir of fython" && exit 1
+  [[ -z "$DESTINE_PATH_COMP" ]] && echo "Missing destine path of bootstrap" && exit 1
+  [[ -z "$PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP" ]] && echo "Missing path of previous fython to use as bootstraper" && exit 1
+  [[ -z "$ELIXIR_BEAMS_PATH_COMP" ]] && echo "Missing elixir beams path" && exit 1
 
   ALL_FILES_PATH_COMP=$(find $SRC_DIR_COMP -name '*.fy')
 
   echo "Destine folder: $DESTINE_PATH_COMP"
-  rm -rf $DESTINE_PATH_COMP
-  mkdir $DESTINE_PATH_COMP
+  rm -rf $DESTINE_PATH_COMP/*
 
   cd $ELIXIR_BEAMS_PATH_COMP && cp -r . "$DESTINE_PATH_COMP" && cd /
 
+  EXIT_CODES_PATH=/tmp/bootstra_exit_codes
+  rm -rf $EXIT_CODES_PATH
+  mkdir $EXIT_CODES_PATH
+
   for FILE_PATH in $ALL_FILES_PATH_COMP; do
-      ERL_COMMAND_CALL="application:start(compiler), application:start(elixir), 'Elixir.Code':compiler_options(#{ignore_module_conflict => true}), 'Fython.Core.Code':compile_project_file(<<"'"'$SRC_DIR_COMP'"'">>, <<"'"'${FILE_PATH}'"'">>, "'"'$DESTINE_PATH_COMP'"'"), init:stop().";
-      erl -pa $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP  -noshell -eval "$ERL_COMMAND_CALL";
+      ERL_COMMAND_CALL="application:start(compiler), application:start(elixir), 'Elixir.Code':compiler_options(#{ignore_module_conflict => true}), 'Fython.Core.Code':compile_project_file(<<"'"'$SRC_DIR_COMP'"'">>, <<"'"'${FILE_PATH}'"'">>, "'"'$DESTINE_PATH_COMP'"'", true), init:stop().";
+      FILE_PATH_SCAPED=$(echo $FILE_PATH | sed 's/\//SEPARATOR/g')
+
+      if [ "$COMPILE_IN_PARALEL" = true ] ; then
+        erl -pa $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP  -noshell -eval "$ERL_COMMAND_CALL" || echo $? > $EXIT_CODES_PATH/$FILE_PATH_SCAPED &
+      else
+        erl -pa $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP  -noshell -eval "$ERL_COMMAND_CALL" || echo $? > $EXIT_CODES_PATH/$FILE_PATH_SCAPED
+      fi
+  done
+  wait
+
+  # when running in parallel we need this to ensure that all comands returned 0 code
+  # otherwise this script would return 0 code even when some erl command failed
+  for RESULT in `ls $EXIT_CODES_PATH`; do
+    ORIGINAL_PATH=$(echo $RESULT | sed 's/SEPARATOR/\//g')
+    RESULT=$(cat $EXIT_CODES_PATH/$RESULT)
+    if [[ $RESULT -ne "0" ]]; then
+       echo "Bootstrap Failed --------------------"
+       echo "Failed to compile files: $ORIGINAL_PATH"
+       echo "-------------------------------------"
+       exit 1
+    fi
   done
+  echo "Bootstraped with success"
 }
 
 $*

devtools/bootstrap_old.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -eo pipefail # stop if any error
+
+compile () {
+  # We use the _COMP sufix to avoid conflict with external variables
+  # (variabls of the shell that is running this script)
+  SRC_DIR_COMP=$1
+  DESTINE_PATH_COMP=$2
+  PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP=$3
+  ELIXIR_BEAMS_PATH_COMP=$4
+  COMPILE_IN_PARALEL=true
+
+  ALL_FILES_PATH_COMP=$(find $SRC_DIR_COMP -name '*.fy')
+
+  echo "Destine folder: $DESTINE_PATH_COMP"
+  rm -rf $DESTINE_PATH_COMP
+  mkdir $DESTINE_PATH_COMP
+
+  cd $ELIXIR_BEAMS_PATH_COMP && cp -r . "$DESTINE_PATH_COMP" && cd /
+
+  EXIT_CODES_PATH=/tmp/bootstra_exit_codes
+  rm -rf $EXIT_CODES_PATH
+  mkdir $EXIT_CODES_PATH
+
+  for FILE_PATH in $ALL_FILES_PATH_COMP; do
+      ERL_COMMAND_CALL="application:start(compiler), application:start(elixir), 'Elixir.Code':compiler_options(#{ignore_module_conflict => true}), 'Fython.Core.Code':compile_project_file(<<"'"'$SRC_DIR_COMP'"'">>, <<"'"'${FILE_PATH}'"'">>, "'"'$DESTINE_PATH_COMP'"'"), init:stop().";
+      FILE_PATH_SCAPED=$(echo $FILE_PATH | sed 's/\//SEPARATOR/g')
+
+      if [ "$COMPILE_IN_PARALEL" = true ] ; then
+        erl -pa $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP  -noshell -eval "$ERL_COMMAND_CALL" || echo $? > $EXIT_CODES_PATH/$FILE_PATH_SCAPED &
+      else
+        erl -pa $PATH_FYTHON_TO_USE_AS_BOOTSTRAPER_COMP  -noshell -eval "$ERL_COMMAND_CALL" || echo $? > $EXIT_CODES_PATH/$FILE_PATH_SCAPED
+      fi
+  done
+  wait
+
+  # when running in parallel we need this to ensure that all comands returned 0 code
+  # otherwise this script would return 0 code even when some erl command failed
+  for RESULT in `ls $EXIT_CODES_PATH`; do
+    ORIGINAL_PATH=$(echo $RESULT | sed 's/SEPARATOR/\//g')
+    RESULT=$(cat $EXIT_CODES_PATH/$RESULT)
+    if [[ $RESULT -ne "0" ]]; then
+       echo "Bootstrap Failed --------------------"
+       echo "Failed to compile files: $ORIGINAL_PATH"
+       echo "-------------------------------------"
+       exit 1
+    fi
+  done
+  echo "Bootstraped with success"
+}
+
+$*

devtools/fython

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e # stop if any error
+set -eo pipefail # stop if any error
 
 exec () {
   FYTHON_CODE=$1
@@ -9,7 +9,7 @@ exec () {
   for ERL_PATH in $ERLANG_PATHS; do
     ERLANG_PATHS_FORMATED=$ERLANG_PATHS_FORMATED"-pa $ERL_PATH "
   done
-  erl $ERLANG_PATHS_FORMATED -noshell -eval "'Fython.Core':eval_string(<<"'"'"<stdin>"'"'">>, <<"'"'"$FYTHON_CODE"'"'">>)." -run init stop
+  erl $ERLANG_PATHS_FORMATED -noshell -eval "'Fython.Core':eval_string(<<"'"'"$FYTHON_CODE"'"'">>)." -run init stop
 }
 
 $*

libs/fytest/__init__.fy

@@ -0,0 +1,45 @@
+def run(path):
+    # TODO it should compile the tests module
+
+    test_functions_per_module = Fytest.Discover.find_test_functions(path)
+    Elixir.IO.puts('Colleted tests:')
+    Elixir.IO.inspect(test_functions_per_module)
+    run_tests(test_functions_per_module)
+
+def run_tests(test_functions_per_module):
+    results_per_module = test_functions_per_module
+        |> Elixir.Enum.map(lambda (module, functions):
+            results = functions
+                |> Elixir.Enum.map(
+                    lambda (test_name, arity):
+                        execute_test(module, test_name, arity)
+                )
+            (module, results)
+        )
+
+    results_per_module
+        |> Elixir.Enum.map(lambda (module, results):
+            show_results(results)
+        )
+
+def show_results(results):
+    passed = results
+        |> Elixir.Enum.filter(lambda (status, _, _, _, _): status == :passed)
+        |> Elixir.Enum.map(lambda (_, module, test_name, _, _):
+            Elixir.IO.puts(Elixir.Enum.join(["PASSED: ", module, test_name]))
+        )
+
+    Elixir.IO.puts('=========================')
+
+    failed = results
+        |> Elixir.Enum.filter(lambda (status, _, _, _, _): status == :failed)
+        |> Elixir.Enum.map(lambda (_, module, test_name, error, stacktrace):
+            Elixir.IO.puts(Elixir.Enum.join(["FAILED: ", module, test_name]))
+        )
+
+def execute_test(module, test_name, _arity):
+    try:
+        Elixir.Kernel.apply(module, test_name, [])
+        (:passed, module, test_name, None, None)
+    except error:
+        (:failed, module, test_name, error, __STACKTRACE__)

libs/fytest/discover/__init__.fy

@@ -0,0 +1,40 @@
+def get_module_name_of_path(path):
+    path
+        |> Elixir.String.replace_suffix('/', '')
+        |> Elixir.String.split('/')
+        |> Elixir.Enum.at(-1)
+        |> Elixir.Macro.camelize()
+
+def find_test_functions(test_module):
+    modules = get_modules(test_module)
+
+    modules
+        |> Elixir.Enum.map(
+            lambda module:
+                functions = module |> get_tests_of_module()
+                (module, functions)
+        )
+
+def get_tests_of_module(module):
+    module
+        |> get_functions_of_module()
+        |> Elixir.Enum.filter(lambda (func_name, _arity):
+            func_name
+                |> Elixir.Atom.to_string()
+                |> Elixir.String.starts_with?("test_")
+        )
+
+def get_functions_of_module(module):
+    # get_functions_of_module(:"Fython.Shell")
+    # TODO go to core
+    Elixir.Kernel.apply(module, :__info__, [:functions])
+
+def get_modules(path):
+    [path, "**/*.fy"]
+        |> Elixir.Enum.join('/')
+        |> Elixir.Path.wildcard()
+        |> Elixir.Enum.map(lambda file_full_path:
+            module_name = Core.Code.get_module_name(path, file_full_path)
+            (:module, module) = Elixir.Code.ensure_loaded(Elixir.String.to_atom(module_name))
+            module
+        )