[DGFiP] necessary adjustments to build shared object as used by DGFiP…

… API (#216)

Part 1 of the tax calculator deliveries to DGFiP clients : tax calculator as a shared object for internal APIs.
Part 2 of the tax calculator deliveries to DGFiP clients : tax calculator as an archive for "modern" mainframes.

New parameters and options for makefiles to improve versatility of the builds.

Makefile.config.template

@@ -47,7 +47,22 @@
 ##################################################
 # C backend configuration
 ##################################################
-# CC=gcc
+# CC=clang
+
+# Optimization level is selected based on compiler. Other flags may be set here.
+# COMMON_CFLAGS=-std=c89 -pedantic
+
+# Mlang includes a light C interface and a light Ocaml driver to build an autotest application.
+# "Autotest" is a standalone CLI tax calculator using FIP files as its data format and limited to
+# initial tax computation.
+# STATIC_DIR?=static/
+# DRIVER_DIR?=ml_driver/
+# DRIVER_FILES?=stubs.c common.ml m.ml read_test.ml main.ml
+
+# C backend test harness features a binary comparison between the dumped computed raw data structure
+# and a reference dump. This flag disables it, falling back to the FIP specified test results only.
+# Useful when you don't have reference dumps.
+# NO_BINARY_COMPARE=1
 
 ##################################################
 # Exports 

Makefile.include

@@ -2,7 +2,11 @@
 # Local customization
 ##################################################
 
-SELF_DIR=$(realpath $(dir $(realpath $(lastword $(MAKEFILE_LIST)))))
+# WARNING: Use of simple expansion here (":=" operator) is important to ensure the value of SELF_DIR
+#          is not evaluated again after inclusion of other makefiles. If these files are in 
+#          another directory, this directory would have been the new SELF_DIR after them. You probably
+#          do not want that. Replace by "=" only if you understand this paragraph and know you need it.
+SELF_DIR:=$(realpath $(dir $(realpath $(lastword $(MAKEFILE_LIST)))))
 
 -include $(SELF_DIR)/Makefile.config
 
@@ -14,12 +18,12 @@ define source_dir
 	$(shell find $(1) -name \*.m ! -name err\*.m ! -name tgv\*.m | sort) $(1)errI.m $(1)tgvI.m
 endef
 
-SOURCE_DIR_2015=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2015m_4_6/)
-SOURCE_DIR_2016=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2016m_4_5/)
-SOURCE_DIR_2017=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2017m_6_10/)
-SOURCE_DIR_2018=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2018m_6_7/)
-SOURCE_DIR_2019=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2019m_8_0/)
-SOURCE_DIR_2020=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2020m_6_5/)
+SOURCE_DIR_2015:=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2015m_4_6/)
+SOURCE_DIR_2016:=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2016m_4_5/)
+SOURCE_DIR_2017:=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2017m_6_10/)
+SOURCE_DIR_2018:=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2018m_6_7/)
+SOURCE_DIR_2019:=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2019m_8_0/)
+SOURCE_DIR_2020:=$(call source_dir,$(SELF_DIR)/ir-calcul/sources2020m_6_5/)
 
 YEAR?=2020
 
@@ -75,13 +79,34 @@ ifeq ($(origin CC),default)
     CC=clang
 endif
 
+# Options pour le compilateur C
+# Attention, très long à compiler avec GCC en O2/O3
+COMMON_CFLAGS?=-std=c89 -pedantic
+ifeq ($(CC), clang)
+    COMPILER_SPECIFIC_CFLAGS=-O2
+else ifeq ($(CC), gcc)
+    COMPILER_SPECIFIC_CFLAGS=-O1
+endif
+BACKEND_CFLAGS=$(COMMON_CFLAGS) $(COMPILER_SPECIFIC_CFLAGS)
+
+# Directory of the static C sources for tax calculator interface
+STATIC_DIR?=static/
+
+# Directory of the driver sources for tax calculator
+DRIVER_DIR?=ml_driver/
+# Driver sources for tax calculator (must be manually ordered for OCaml compiler)
+DRIVER_FILES?=stubs.c common.ml m.ml read_test.ml main.ml
+
+# Flag to disable binary dump comparison
+NO_BINARY_COMPARE?=0
+
 ##################################################
 # Exports to call backends from main Makefile
 ##################################################
 
 # common
 export SOURCE_FILES TESTS_DIR MLANG_BIN MLANG_DEFAULT_OPTS
 # for C backend (Java compilation is year-independent)
-export YEAR CC
+export YEAR CC BACKEND_CFLAGS STATIC_DIR DRIVER_DIR DRIVER_FILES NO_BINARY_COMPARE
 # for Java backend (C overload these now)
 export MPP_FUNCTION MPP_FILE

examples/dgfip_c/.gitignore

@@ -10,6 +10,7 @@ ml_primitif/*.cm*
 ml_primitif/cal
 ml_primitif/vars.txt
 ml_primitif/*.output
+ml_primitif/*.so
 
 fuzz_findings/*
 fuzz_tests/*

examples/dgfip_c/ml_primitif/Makefile

@@ -16,23 +16,26 @@ MLANG_DGFIP_C_OPTS=\
 	--mpp_file=$(MPP_FILE) \
 	--mpp_function=$(MPP_FUNCTION)
 
-# Options supplémentaires pour le backend Mlang/DGFiP
-# Note: les flags -Ailiad, -m, -g et -k sont déjà définis
-DGFIPFLAGS=-X,-O  # = environnement DGFiP
+# Options supplémentaires pour le backend Mlang/DGFiP spécifiques à la cible
+# Par défaut ici pour l'autotest Mlang et l'intégration continue.
+# Note : compilation avec -g et -k OBLIGATOIREMENT
+#  -g  : informations pour tests et débogage
+#  -kN : nombre de segment pour la table de débogage.
+#        N=1 à 4 pour le pilotage Mlang ml_primitif
+#        N=4 pour utiliser le pilotage DGFiP.
+# Note : compilation avec -O (optimisation par inlining)
+DGFIP_TARGET_FLAGS?=-g,-O,-k4
+# Options supplémentaires pour le backend Mlang/DGFiP communes à tous les build :
+# Note: les flags -Ailiad, -g et -k sont déjà définis
+# -Ailiad : ensemble de règles sélectionné ("Application ciblée")
+# -m : millésime de calculette compilé (année des revenus taxés)
+# -X : génération de fonctions d'extraction globale dans l'interface,
+#      bouclant sur la table des variables restituables (IN_init_extraction).
+DGFIP_COMMON_FLAGS=-Ailiad,-m$(YEAR),-X
 
 MLANG=$(MLANG_BIN) $(MLANG_DEFAULT_OPTS) $(MLANG_DGFIP_C_OPTS)
 QUIET=>/dev/null # Uncomment to suppress output
 
-# Options pour le compilateur C
-# Attention, très long à compiler avec GCC en O2/O3
-COMMON_CFLAGS=-std=c89 -pedantic
-ifeq ($(CC), clang)
-    COMPILER_SPECIFIC_CFLAGS=-O2
-else ifeq ($(CC), gcc)
-    COMPILER_SPECIFIC_CFLAGS=-O1
-endif
-CFLAGS=$(COMMON_CFLAGS) $(COMPILER_SPECIFIC_CFLAGS)
-
 # Options pour le compilateur OCaml
 OCAMLFLAGS=
 #OCAMLFLAGS="-g -inline 0"
@@ -54,20 +57,17 @@ endif
 # Generating C files from M sources with Mlang
 ##################################################
 
-calc/var_static.c.inc calc/enchain_static.c.inc calc/desc_static.h.inc calc/const.h calc/dbg.h calc/irdata.c calc/irdata.h &:
+# TODO: use &: when upgraded to GNU Make 4.3+
+calc/var_static.c.inc calc/enchain_static.c.inc calc/desc_static.h.inc calc/const.h calc/dbg.h calc/irdata.c calc/irdata.h :
 	mkdir -p calc
-	@echo "Copie des fichiers C statiques"
-	cp static/* calc/
+	@echo "Copie des fichiers C statiques depuis $(STATIC_DIR)"
+	cp $(STATIC_DIR)* calc/
 
-# Ml_primitif (current main build)
-# -----------------------------
-# Note : compilation avec -g et -kN (N=1..4) OBLIGATOIREMENT
-# Note : la compilation mainframe en 2022 a pu se faire avec -rXMk4,-b0,-Ailiad ?
 calc/enchain.c: $(SOURCE_FILES)
 	mkdir -p calc
-	@echo "Compilation des fichiers M avec Mlang (YEAR=$(YEAR), MPP_FUNCTION=$(MPP_FUNCTION), DGFIPFLAGS=$(DGFIPFLAGS))"
+	@echo "Compilation des fichiers M avec Mlang (MPP_FUNCTION=$(MPP_FUNCTION), DGFIP_TARGET_FLAGS=$(DGFIP_TARGET_FLAGS), DGFIP_COMMON_FLAGS=$(DGFIP_COMMON_FLAGS))"
 	@$(MLANG) \
-		--dgfip_options=-g,-k4,-Ailiad,-m$(YEAR),$(DGFIPFLAGS)\
+		--dgfip_options=$(DGFIP_TARGET_FLAGS),$(DGFIP_COMMON_FLAGS)\
 		--backend dgfip_c --output $@ \
 		$(SOURCE_FILES) $(QUIET)
 	@echo "Compilation terminée"
@@ -79,30 +79,36 @@ calc/enchain.c: $(SOURCE_FILES)
 # list existing C file from M equivalent (Useful idea to improve upon)
 M_C_FILES=$(filter-out tgv%.c err%.c,$(notdir $(SOURCE_FILES:.m=.c)))
 
-# Ml_primitif (current main build)
-# -----------------------------
-calc/enchain.o: calc/var_static.c.inc calc/enchain.c
-	@echo "Compilation des fichiers C issus des fichiers M (CC=$(CC), CFLAGS=$(CFLAGS))"
-	cd calc && $(CC) $(CFLAGS) -c \
-	irdata.c enchain.c var.c contexte.c famille.c revenu.c revcor.c penalite.c variatio.c tableg??.c restitue.c \
+# ADDITIONAL_C_SOURCES_TARGETS allows to create new targets in configuration to produce or load in the source folder before compiling.
+calc/enchain.o: calc/var_static.c.inc calc/enchain.c $(ADDITIONAL_C_SOURCES_TARGETS)
+	@echo "Compilation des fichiers C issus des fichiers M (CC=$(CC), BACKEND_CFLAGS=$(BACKEND_CFLAGS))"
+	cd calc && $(CC) $(BACKEND_CFLAGS) -c \
+	*.c
+#	irdata.c enchain.c var.c contexte.c famille.c revenu.c revcor.c penalite.c variatio.c tableg??.c restitue.c \
 	chap-*.c res-ser*.c coc*.c coi*.c horiz*.c
 
 #################################################
 # Final targets
 ##################################################
 
+DRIVER_TARGETS:=$(foreach file,$(DRIVER_FILES),calc/$(file))
+# TODO: use &: when upgraded to GNU Make 4.3+
+$(DRIVER_TARGETS) :
+	echo "Copie des sources du pilote depuis $(DRIVER_DIR)"
+	cp $(DRIVER_DIR)* calc/
+
 # Ml_primitif (current main build)
 # -----------------------------
-cal: calc/enchain.o
+cal: calc/enchain.o $(DRIVER_TARGETS)
 	@echo "Compilation de la calculette primitive (OCAMLFLAGS=$(OCAMLFLAGS), WITH_BISECT=$(WITH_BISECT))"
 ifeq ($(WITH_BISECT), 1)
-	ocamlopt -cc $(CC) -ccopt -std=c99 -ccopt -fno-common \
+	cd calc && ocamlopt -cc $(CC) -ccopt -std=c99 -ccopt -fno-common \
 	-I $(BISECT_PATH)/common -I $(BISECT_PATH)/runtime \
 	-ppx "$(BISECT_PATH)/ppx.exe --as-ppx" \
-	unix.cmxa bisect_common.cmxa bisect.cmxa ./calc/*.o stubs.c common.ml m.ml read_test.ml main.ml -o cal
+	unix.cmxa bisect_common.cmxa bisect.cmxa *.o $(DRIVER_FILES) -o cal
 else
-	ocamlopt -cc $(CC) $(OCAMLFLAGS) -ccopt -std=c99 -ccopt -fno-common \
-	unix.cmxa ./calc/*.o stubs.c common.ml m.ml read_test.ml main.ml -o cal
+	cd calc && ocamlopt -cc $(CC) $(OCAMLFLAGS) -ccopt -std=c99 -ccopt -fno-common \
+	unix.cmxa *.o $(DRIVER_FILES) -o ../cal
 endif
 
 ##################################################
@@ -123,18 +129,19 @@ backend_tests: FORCE run_tests
 clean:
 	@echo "Nettoyage des fichiers binaires intermédiaires"
 	rm -f calc/*.o
-	rm -f *.o
-	rm -f *.cm*
+	rm -f calc/*.cm*
 
 # To keep in mind
 # rm -f $(M_C_FILES) $(M_C_FILES:.c=.o) \
 	contexte.* famille.* penalite.* restitue.* revcor.* \
 	revenu.* tableg*.* tablev.* variatio.* var.* \
 	conf.h annee.h desc.h desc_inv.h
 cleanc:
-	@echo "Nettoyage des sources C"
+	@echo "Nettoyage des sources"
 	rm -f calc/*.[ch]
 	rm -f calc/*.inc
+	rm -f calc/version.*
+	rm -f calc/*.ml
 
 cleanexe:
 	@echo "Nettoyage des exécutables"

examples/dgfip_c/ml_primitif/main.ml → examples/dgfip_c/ml_primitif/ml_driver/main.ml

@@ -118,7 +118,7 @@ let compare_dump out outexp =
   close_in out;
   close_in outexp
 
-let run_test test_file =
+let run_test test_file flag_no_bin_compare =
   Printf.printf "Testing %s...\n%!" test_file;
 
   let annee_calc = M.annee_calc () in
@@ -157,26 +157,29 @@ let run_test test_file =
 
   let res_ok = check_result tgv err res_prim ctl_prim in
 
-  if not (Sys.file_exists out_exp)
-  then begin
-    Printf.eprintf "Expected results file %s not found." out_exp;
-    exit (if res_ok then 10 else 11)
-  end;
-
-  begin
-    match Unix.system (Printf.sprintf "diff -q %s %s 1>/dev/null 2>&1" out out_exp) with
-    | WEXITED 0 ->
-        if res_ok then 0 else 1
-    | WEXITED 1 -> (* Differences *)
-        compare_dump out out_exp;
-        if res_ok then 20 else 21
-    | WEXITED 2 -> (* Other error *)
-        if res_ok then 30 else 31
-    | WEXITED _ ->
-        32
-    | WSIGNALED _ | WSTOPPED _ ->
-        33
-  end
+  match flag_no_bin_compare with
+  | true -> if res_ok then 0 else 1
+  | false -> 
+    if not (Sys.file_exists out_exp)
+      then begin
+      Printf.eprintf "Expected results file %s not found." out_exp;
+      exit (if res_ok then 10 else 11)
+    end;
+
+    begin
+      match Unix.system (Printf.sprintf "diff -q %s %s 1>/dev/null 2>&1" out out_exp) with
+      | WEXITED 0 ->
+          if res_ok then 0 else 1
+      | WEXITED 1 -> (* Differences *)
+          compare_dump out out_exp;
+          if res_ok then 20 else 21
+      | WEXITED 2 -> (* Other error *)
+          if res_ok then 30 else 31
+      | WEXITED _ ->
+          32
+      | WSIGNALED _ | WSTOPPED _ ->
+          33
+      end
 
 let main () =
   if Array.length Sys.argv < 2 then
@@ -189,11 +192,17 @@ let main () =
          0o755 with _ -> ());
 
   let args = List.tl (Array.to_list Sys.argv) in
+  let flag_no_bin_compare = 
+    match Sys.getenv_opt "NO_BINARY_COMPARE" with
+    | Some "1" -> true
+    | None | Some "0"-> false
+    | _ -> exit 31
+  in
   let rec loop =
     function
     | [] -> 0
     | test_file::args ->
-        let res = run_test test_file in
+        let res = run_test test_file flag_no_bin_compare in
         if res <> 0 then res else loop args
   in
   loop args

examples/dgfip_c/ml_primitif/stubs.c → examples/dgfip_c/ml_primitif/ml_driver/stubs.c

@@ -14,12 +14,12 @@
 #include "caml/fail.h"
 #include "caml/custom.h"
 
-#include "calc/annee.h"
-#include "calc/conf.h"
-#include "calc/irdata.h"
-#include "calc/const.h"
-#include "calc/var.h"
-#include "calc/enchain.h"
+#include "annee.h"
+#include "conf.h"
+#include "irdata.h"
+#include "const.h"
+#include "var.h"
+#include "enchain.h"
 
 #if OCAML_VERSION < 41200