Significant update to build and install process for ccf ledger

Several parts of this commit:

1) move the ccf python scripts into their own directory and package
them into an installable python wheel file; this should lay the
structural foundation for supporting our own version of the CCF python
client (which is not included in more recent versions of CCF).

2) canonicalize argument handling for the ccf python scripts; this
moves all argument processing (including the common handling of
creating the CCF client) into a single routine. All CCF python files
should share the same structure and command line API.

3) split the build of the CCF ledger components into two pieces. The
first piece builds and installs the python packages. Since these utilities
are useful on any PDO node (client, service, ccf ledger), they can now
be installed independently from the pdo tp. The second piece is the PDO
TP itself which need only be installed on ccf ledger nodes.

Signed-off-by: Mic Bowman <[email protected]>

build/cmake/ProjectVariables.cmake

@@ -55,3 +55,25 @@ ELSE()
   ADD_COMPILE_OPTIONS(-Wno-deprecated)
   ADD_COMPILE_OPTIONS(-Wno-deprecated-declarations)
 ENDIF()
+
+IF (NOT DEFINED ENV{PDO_INSTALL_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_INSTALL_ROOT not defined")
+ENDIF()
+SET(PDO_INSTALL_ROOT $ENV{PDO_INSTALL_ROOT})
+
+IF (NOT DEFINED ENV{PDO_SOURCE_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_SOURCE_ROOT not defined")
+ENDIF()
+SET(PDO_SOURCE_ROOT $ENV{PDO_SOURCE_ROOT})
+
+# Get the current version using the get_version
+# utility; note that this will provide 0.0.0 as
+# the version if something goes wrong (like running
+# without any annotated version tags)
+EXECUTE_PROCESS(
+  COMMAND ./get_version
+  WORKING_DIRECTORY ${PDO_SOURCE_ROOT}/bin
+  OUTPUT_VARIABLE PDO_VERSION
+  ERROR_QUIET
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)

build/cmake/Python.cmake

@@ -0,0 +1,53 @@
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# assumes that project variables have been
+# set including PDO_INSTALL_ROOT and PDO_CONTRACT_VERSION
+
+IF (NOT DEFINED PDO_INSTALL_ROOT)
+  MESSAGE(FATAL_ERROR "PDO_INSTALL_ROOT not defined")
+ENDIF()
+
+IF (NOT DEFINED PDO_VERSION)
+  MESSAGE(FATAL_ERROR "PDO_VERSION not defined")
+ENDIF()
+
+SET(PIP "${PDO_INSTALL_ROOT}/bin/pip3" CACHE STRING "Pip executable in virtual environment")
+SET(PYTHON "${PDO_INSTALL_ROOT}/bin/python3" CACHE STRING "Python executable in virtual environment")
+SET(RESOURCE_INSTALLER "${PDO_INSTALL_ROOT}/bin/pdo-install-plugin-resources" CACHE STRING "PDO resource installer")
+SET(WHEEL_PATH "${CMAKE_BINARY_DIR}/dist" CACHE STRING "Path where python wheels will be placed")
+
+FUNCTION(BUILD_WHEEL package)
+  SET(SOURCE ${CMAKE_CURRENT_SOURCE_DIR})
+  SET(WHEEL_FILE "${WHEEL_PATH}/${package}-${PDO_VERSION}-py3-none-any.whl")
+  FILE(STRINGS "${SOURCE}/MANIFEST" MANIFEST)
+
+  # adding the build and egg-info directories to the output means that
+  # they will be cleaned up with the global clean target
+  ADD_CUSTOM_COMMAND(
+    OUTPUT ${WHEEL_FILE} ${SOURCE}/${package}.egg-info
+    COMMAND ${PYTHON}
+    ARGS -m build --wheel --outdir ${WHEEL_PATH}
+    WORKING_DIRECTORY ${SOURCE}
+    DEPENDS ${MANIFEST})
+
+  ADD_CUSTOM_TARGET(${package}-package ALL DEPENDS ${WHEEL_FILE})
+
+  STRING(JOIN "\n" INSTALL_COMMAND
+    "MESSAGE(\"INSTALL ${package}\")"
+    "EXECUTE_PROCESS(COMMAND ${PIP} uninstall --yes ${WHEEL_FILE})"
+    "EXECUTE_PROCESS(COMMAND ${PIP} install ${WHEEL_FILE})" )
+
+  INSTALL(CODE ${INSTALL_COMMAND})
+ENDFUNCTION()

ledgers/ccf/.gitignore

@@ -1 +1 @@
-build
+build*

ledgers/ccf/CMakeLists.txt

@@ -14,16 +14,15 @@
 
 CMAKE_MINIMUM_REQUIRED(VERSION 3.16)
 
-INCLUDE(${CCF_DIR}/cmake/preproject.cmake)
-PROJECT(pdoenc LANGUAGES C CXX)
-INCLUDE(${CCF_DIR}/cmake/ccf_app.cmake)
+IF (NOT DEFINED ENV{PDO_SOURCE_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_SOURCE_ROOT not defined")
+ENDIF()
+SET(PDO_SOURCE_ROOT $ENV{PDO_SOURCE_ROOT})
 
-FIND_PACKAGE(ccf_virtual REQUIRED)
+LIST(APPEND CMAKE_MODULE_PATH "${PDO_SOURCE_ROOT}/build/cmake")
+INCLUDE(ProjectVariables)
 
-# Add the PDO transaction processor target
-add_ccf_app(pdoenc
-  SRCS transaction_processor/pdo_tp.cpp transaction_processor/verify_signatures.cpp
-  INCLUDE_DIRS ${CCF_DIR}/include/ccf ${CCF_DIR}/include/3rdparty ${CCF_DIR}/include
-)
+PROJECT(pdo_ccf_ledger)
 
-INSTALL(TARGETS pdoenc.virtual DESTINATION lib)
+INCLUDE(Python)
+BUILD_WHEEL(pdo_ccf_ledger)

ledgers/ccf/MANIFEST

@@ -0,0 +1,12 @@
+MANIFEST.in
+setup.py
+pdo/__init__.py
+pdo/ledgers/__init__.py
+pdo/ledgers/ccf/__init__.py
+pdo/ledgers/ccf/common.py
+pdo/ledgers/ccf/scripts/__init__.py
+pdo/ledgers/ccf/scripts/fetch_ledger_authority.py
+pdo/ledgers/ccf/scripts/ping_test.py
+pdo/ledgers/ccf/scripts/register_enclave_attestation_verification_policy.py
+pdo/ledgers/ccf/scripts/generate_ledger_authority.py
+pdo/ledgers/ccf/scripts/configure_ccf_network.py

ledgers/ccf/Makefile

@@ -35,7 +35,14 @@ CCF_LEDGER_DIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
 NINJA ?= ninja $(NINJA_OPTIONS)
 
 SCRIPTDIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-BLDDIR := $(SCRIPTDIR)/build
+
+# Build the PDO TP in the transaction processor subdirectory
+PDO_TP_SRCDIR := $(SCRIPTDIR)/transaction_processor
+PDO_TP_BLDDIR := $(SCRIPTDIR)/transaction_processor/build
+
+# Build the Python wheel in this directory
+PYTHON_SRCDIR := $(SCRIPTDIR)
+PYTHON_BLDDIR := $(SCRIPTDIR)/build
 
 all: environment install
 
@@ -76,22 +83,28 @@ $(CCF_LEDGER_DIR)/lib/python3.8 : pdo-environment
 # -----------------------------------------------------------------
 build : build-pdo-tp
 
-build-pdo-tp : $(BLDDIR)
-	cmake --build $(BLDDIR)
+build-pdo-tp : $(PDO_TP_BLDDIR)
+	cmake --build $(PDO_TP_BLDDIR)
 
-$(BLDDIR) :
-	cmake -S . -B $(BLDDIR) -GNinja \
+$(PDO_TP_BLDDIR) :
+	cmake -S $(PDO_TP_SRCDIR) -B $(PDO_TP_BLDDIR) -GNinja \
 		-DCCF_DIR=$(CCF_BASE) \
 		-DCOMPILE_TARGET=$(CCF_COMPILE_TARGET) \
 		-DCMAKE_INSTALL_PREFIX=$(CCF_LEDGER_DIR)
 
+build-python : $(PYTHON_BLDDIR)
+	cmake --build $(PYTHON_BLDDIR)
+
+$(PYTHON_BLDDIR) :
+	cmake -S $(PYTHON_SRCDIR) -B $(PYTHON_BLDDIR)
+
 # -----------------------------------------------------------------
 # clean up
 # -----------------------------------------------------------------
 clean : clean-build clean-ledger
 
 clean-build:
-	rm -rf $(BLDDIR)
+	rm -rf $(PDO_TP_BLDDIR) $(PYTHON_BLDDIR)
 
 clean-ledger :
 	rm -rf $(CCF_LEDGER_DIR)
@@ -109,15 +122,14 @@ clean-ledger :
 PDO_BASH_SCRIPTS = $(wildcard scripts/*.sh)
 PDO_PYTHON_SCRIPTS = $(wildcard scripts/*.py)
 
-install : install-pdo-tp install-pdo-scripts
+install : install-pdo-tp install-python
 
-# cd $(BLDDIR) && $(NINJA) install
 install-pdo-tp : ledger-environment build-pdo-tp
-	cmake --install $(BLDDIR)
+	cmake --install $(PDO_TP_BLDDIR)
 	cp $(PDO_BASH_SCRIPTS) $(CCF_LEDGER_DIR)/bin
 
-install-pdo-scripts : pdo-environment
-	cp $(PDO_PYTHON_SCRIPTS) $(PDO_INSTALL_ROOT)/bin
+install-python : pdo-environment build-python
+	cmake --install $(PYTHON_BLDDIR)
 
 # -----------------------------------------------------------------
 .PHONY : all

ledgers/ccf/pdo/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__import__('pkg_resources').declare_namespace('pdo')

ledgers/ccf/pdo/ledgers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ledgers/ccf/pdo/ledgers/ccf/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ledgers/ccf/pdo/ledgers/ccf/common.py

@@ -0,0 +1,144 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+from urllib.parse import urlparse
+
+from ccf.clients import Identity
+from ccf.clients import CCFClient
+
+from loguru import logger as LOG
+
+# -----------------------------------------------------------------
+# parse options and initialize the common variables
+# -----------------------------------------------------------------
+def parse_common_arguments(args, description, member_keys_required = False) :
+
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        '--logfile',
+        help='Name of the log file, __screen__ for standard output',
+        default='__screen__',
+        type=str)
+
+    parser.add_argument(
+        '--loglevel',
+        help='Logging level',
+        default='WARNING',
+        type=str)
+
+    parser.add_argument(
+        '--url',
+        help='URL for the ledger',
+        default = os.environ.get("PDO_LEDGER_URL"),
+        type=str)
+
+    parser.add_argument(
+        '--interface',
+        help='Host interface where CCF is listening',
+        type=str)
+
+    parser.add_argument(
+        '--port',
+        help='Port where CCF is listening',
+        type=int,
+        default=6600)
+
+    parser.add_argument(
+        '--key-dir',
+        help='Directory where certificate files are located, defaults to PDO_LEDGER_KEY_ROOT',
+        default=os.environ.get("PDO_LEDGER_KEY_ROOT"),
+        type=str)
+
+    parser.add_argument(
+        '--cert',
+        help='Name of the network certificate file',
+        type=str,
+        default='networkcert.pem')
+
+    parser.add_argument(
+        '--member',
+        help="Name of the network membership certificate",
+        default = "memberccf",
+        type=str)
+
+    (options, unprocessed_args) = parser.parse_known_args(args)
+
+    # set up the logging
+    LOG.remove()
+    if options.logfile == '__screen__' :
+        LOG.add(sys.stderr, level=options.loglevel)
+    else :
+        LOG.add(options.logfile)
+
+    # precedence is given to ledger interface through the interface/port parameters; the fall back
+    # is to use the ledger url parameter
+    if options.interface :
+        pass
+    elif options.url :
+        (options.interface, options.port) = urlparse(options.url).netloc.split(':')
+    else :
+        LOG.error('no ledger interface specified')
+        sys.exit(-1)
+
+    # the key directory must be specified either through the PDO_LEDGER_KEY_ROOT
+    # environment variable or the key-dir parameter
+    if not options.key_dir or not os.path.exists(options.key_dir) :
+        LOG.error('unable to locate key dir')
+        sys.exit(-1)
+
+    network_cert = os.path.join(options.key_dir, options.cert)
+    if not os.path.exists(network_cert) :
+        LOG.error('network certificate ({}) does not exist'.format(network_cert))
+        sys.exit(-1)
+
+    # now create the client
+    if member_keys_required :
+        member_cert = os.path.join(options.key_dir, "{}_cert.pem".format(options.member))
+        if not os.path.exists(member_cert) :
+            LOG.error('member certificate ({}) does not exist'.format(member_cert))
+            sys.exit(-1)
+
+        member_key = os.path.join(options.key_dir, "{}_privk.pem".format(options.member))
+        if not os.path.exists(member_key) :
+            LOG.error('member key ({}) does not exist'.format(member_key))
+            sys.exit(-1)
+
+        try :
+            client = CCFClient(
+                options.interface,
+                options.port,
+                network_cert,
+                session_auth=Identity(member_key, member_cert, "member"),
+                signing_auth=Identity(member_key, member_cert, "member"),
+            )
+        except Exception as e:
+            LOG.error('failed to connect to CCF service : {}'.format(str(e)))
+            sys.exit(-1)
+
+    else :
+        try :
+            client = CCFClient(
+                options.interface,
+                options.port,
+                network_cert)
+        except Exception as e:
+            LOG.error('failed to connect to CCF service : {}'.format(str(e)))
+            sys.exit(-1)
+
+    # and return the client plus any operation-specific arguments that have not been processed
+    return (options, unprocessed_args, client)

ledgers/ccf/pdo/ledgers/ccf/scripts/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.