diff --git a/README.md b/README.md index 1db8988..a96af0e 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,11 @@ Make sure you have Ansible installed (`pip3 install ansible`), then copy the fol Each host should be reachable via SSH using the username set in `ansible_user`. Other Ansible options can be set under `[cluster:vars]` to connect in more exotic clustering scenarios (e.g. via bastion/jump-host). -Tweak other settings inside `config.yml` as desired (the most important being `hpl_root`—this is where the compiled MPI, ATLAS, and HPL benchmarking code will live). +Tweak other settings inside `config.yml` as desired (the most important being `hpl_root`—this is where the compiled MPI, ATLAS/OpenBLAS/Blis, and HPL benchmarking code will live). -> **Note**: -> The names of the nodes inside `hosts.ini` must match the hostname of their corresponding node; otherwise, the benchmark will hang when you try to run it in a cluster. -> For example, if you have `node-01.local` in your `hosts.ini` your host's hostname should be `node-01` and not something else like `raspberry-pi`. +> **Note**: The names of the nodes inside `hosts.ini` must match the hostname of their corresponding node; otherwise, the benchmark will hang when you try to run it in a cluster. +> +> For example, if you have `node-01.local` in your `hosts.ini` your host's hostname should be `node-01` and not something else like `raspberry-pi`. Then run the benchmarking playbook inside this directory: @@ -59,7 +59,7 @@ This will run three separate plays: After the entire playbook is complete, you can also log directly into any of the nodes (though I generally do things on node 1), and run the following commands to kick off a benchmarking run: ``` -cd ~/tmp/hpl-2.3/bin/rpi +cd ~/tmp/hpl-2.3/bin/top500 mpirun -f cluster-hosts ./xhpl ``` @@ -115,6 +115,7 @@ Here are a few of the results I've acquired in my testing: | DeskPi Super6c (6x CM4 @ 1.5 GHz) | 60.293 Gflops | 40W | 1.50 Gflops/W | | DeskPi Super6c (6x CM4 @ 2.0 GHz) | 70.338 Gflops | 51W | 1.38 Gflops/W | | Radxa ROCK 5B (1x RK3588 8-core) | 46.669 Gflops | 15W | 3.11 Gflops/W | +| Orange Pi 5 (1x RK3588S 8-core) | 53.333 Gflops | 11.7W | 4.56 Gflops/W | | Lenovo M710q Tiny (1x i5-7400T @ 2.4 GHz) | 72.472 Gflops | 41W | 1.76 Gflops/W | | M2 MacBook Air (1x M2 @ 3.5 GHz, in Docker) | 104.68 Gflops | N/A | N/A | | M1 Max Mac Studio (1x M1 Max @ 3.2 GHz, in Docker) | 264.32 Gflops | 66W | 4.00 Gflops/W | @@ -123,3 +124,9 @@ Here are a few of the results I've acquired in my testing: | Ampere Altra Max M128-30 @ 3.0 GHz | 953.47 Gflops | 500W | 1.91 Gflops/W | You can [enter the Gflops in this tool](https://hpl-calculator.sourceforge.net/hpl-calculations.php) to see how it compares to historical top500 lists. + +### Other Listings + +Over the years, as I find other people's listings of HPL results—especially those with power usage ratings—I will add them here: + + - [VMW Research Group GFLOPS/W listing](https://web.eece.maine.edu/~vweaver/group/green_machines.html) diff --git a/dependencies/arch-based.yml b/dependencies/arch-based.yml index 960d9f1..dadeeeb 100755 --- a/dependencies/arch-based.yml +++ b/dependencies/arch-based.yml @@ -7,6 +7,7 @@ - name: Install dependencies. ansible.builtin.package: name: + - git - base-devel - gcc-fortran state: present diff --git a/dependencies/debian-based.yml b/dependencies/debian-based.yml index 92de669..2a52597 100755 --- a/dependencies/debian-based.yml +++ b/dependencies/debian-based.yml @@ -8,6 +8,7 @@ - name: Install dependencies. ansible.builtin.package: name: + - git - build-essential - gfortran - automake diff --git a/example.config.yml b/example.config.yml index adcfe7f..3166753 100644 --- a/example.config.yml +++ b/example.config.yml @@ -2,6 +2,11 @@ # Working directory where HPL and associated applications will be compiled. hpl_root: /opt/top500 +# Linear algebra library options. +linear_algebra_library: atlas # 'atlas', 'openblas', or 'blis' +linear_algebra_blis_version: master # only used for blis +linear_algebra_openblas_version: develop # only used for openblas + # Home directory of the user for whom SSH keys will be configured. ssh_user: pi ssh_user_home: /home/pi diff --git a/main.yml b/main.yml index efc3fed..67b74f5 100644 --- a/main.yml +++ b/main.yml @@ -12,13 +12,13 @@ vars_files: ['config.yml'] tasks: - - include_tasks: dependencies/rhel-based.yml + - ansible.builtin.include_tasks: dependencies/rhel-based.yml when: ansible_os_family == 'RedHat' - - include_tasks: dependencies/debian-based.yml + - ansible.builtin.include_tasks: dependencies/debian-based.yml when: ansible_os_family == 'Debian' - - include_tasks: dependencies/arch-based.yml + - ansible.builtin.include_tasks: dependencies/arch-based.yml when: ansible_os_family == 'Archlinux' - name: Create required temporary directories. @@ -30,7 +30,7 @@ mode: 0755 loop: - "{{ hpl_root }}/tmp" - - "{{ hpl_root }}/tmp/atlas-build" + - "{{ hpl_root }}/tmp/{{ linear_algebra_library }}-build" become: true - name: Download MPI (Message Passing Interface). @@ -72,30 +72,18 @@ # Note: There was no simpler way to do this besides `shell`. - name: Ensure CPU scaling is set to 'performance'. ansible.builtin.shell: >- - echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor when: cpufreq_exists.rc == 0 + become: true - - name: Download ATLAS linear algebra library. - ansible.builtin.unarchive: - src: https://sourceforge.net/projects/math-atlas/files/Stable/3.10.3/atlas3.10.3.tar.bz2 - dest: "{{ hpl_root }}/tmp" - remote_src: true - creates: "{{ hpl_root }}/tmp/ATLAS/README" + - ansible.builtin.include_tasks: tasks/algebra_atlas.yml + when: linear_algebra_library == 'atlas' - - name: Install ATLAS (takes a LONG time). - ansible.builtin.command: "{{ item }}" - args: - chdir: "{{ hpl_root }}/tmp/atlas-build" - creates: "{{ hpl_root }}/tmp/COMPILE_ATLAS_COMPLETE" - loop: - - ../ATLAS/configure - - make + - ansible.builtin.include_tasks: tasks/algebra_blis.yml + when: linear_algebra_library == 'blis' - - name: Create 'COMPILE_ATLAS_COMPLETE' file. - file: - path: "{{ hpl_root }}/tmp/COMPILE_ATLAS_COMPLETE" - state: touch - mode: 0644 + - ansible.builtin.include_tasks: tasks/algebra_openblas.yml + when: linear_algebra_library == 'openblas' - name: Download HPL (High Performance Linpack). ansible.builtin.unarchive: @@ -110,15 +98,15 @@ chdir: "{{ hpl_root }}/tmp/hpl-2.3/setup" creates: "{{ hpl_root }}/tmp/COMPILE_HPL_COMPLETE" - - name: Copy rpi makefile into place. + - name: Copy HPL makefile into place. ansible.builtin.template: - src: templates/benchmark-Make.rpi.j2 - dest: "{{ hpl_root }}/tmp/hpl-2.3/Make.rpi" + src: templates/benchmark-Make.top500.j2 + dest: "{{ hpl_root }}/tmp/hpl-2.3/Make.top500" mode: 0644 - name: Install HPL. ansible.builtin.command: >- - make arch=rpi + make arch=top500 args: chdir: "{{ hpl_root }}/tmp/hpl-2.3" creates: "{{ hpl_root }}/tmp/COMPILE_HPL_COMPLETE" @@ -192,7 +180,7 @@ - name: Create a file describing nodes for MPI execution. ansible.builtin.template: src: templates/mpi-node-config.j2 - dest: "{{ hpl_root }}/tmp/hpl-2.3/bin/rpi/cluster-hosts" + dest: "{{ hpl_root }}/tmp/hpl-2.3/bin/top500/cluster-hosts" mode: 0644 # Template originally generated using this website: @@ -200,7 +188,7 @@ - name: Create HPL.dat file. ansible.builtin.template: src: templates/HPL.dat.j2 - dest: "{{ hpl_root }}/tmp/hpl-2.3/bin/rpi/HPL.dat" + dest: "{{ hpl_root }}/tmp/hpl-2.3/bin/top500/HPL.dat" mode: 0644 # If this is not done, the nodes will fail to connect to each other @@ -211,7 +199,7 @@ - name: Run the benchmark. ansible.builtin.command: mpirun -f cluster-hosts ./xhpl args: - chdir: "{{ hpl_root }}/tmp/hpl-2.3/bin/rpi" + chdir: "{{ hpl_root }}/tmp/hpl-2.3/bin/top500" register: mpirun_output run_once: true diff --git a/tasks/algebra_atlas.yml b/tasks/algebra_atlas.yml new file mode 100644 index 0000000..6c8460c --- /dev/null +++ b/tasks/algebra_atlas.yml @@ -0,0 +1,22 @@ +--- +- name: Download ATLAS linear algebra library. + ansible.builtin.unarchive: + src: https://sourceforge.net/projects/math-atlas/files/Stable/3.10.3/atlas3.10.3.tar.bz2 + dest: "{{ hpl_root }}/tmp" + remote_src: true + creates: "{{ hpl_root }}/tmp/ATLAS/README" + +- name: Install ATLAS (takes a LONG time). + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ hpl_root }}/tmp/atlas-build" + creates: "{{ hpl_root }}/tmp/COMPILE_ATLAS_COMPLETE" + loop: + - ../ATLAS/configure + - make + +- name: Create 'COMPILE_ATLAS_COMPLETE' file. + ansible.builtin.file: + path: "{{ hpl_root }}/tmp/COMPILE_ATLAS_COMPLETE" + state: touch + mode: 0644 diff --git a/tasks/algebra_blis.yml b/tasks/algebra_blis.yml new file mode 100644 index 0000000..26df25d --- /dev/null +++ b/tasks/algebra_blis.yml @@ -0,0 +1,23 @@ +--- +- name: Download Blis linear algebra library. + ansible.builtin.git: + repo: 'https://github.com/flame/blis.git' + version: "{{ linear_algebra_blis_version }}" + dest: "{{ hpl_root }}/tmp/blis-build" + +- name: Install Blis. + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ hpl_root }}/tmp/blis-build" + creates: "{{ hpl_root }}/tmp/COMPILE_BLIS_COMPLETE" + loop: + - ./configure --prefix=/opt/blis auto + - make -j{{ ansible_processor_vcpus }} + - make install + become: true + +- name: Create 'COMPILE_BLIS_COMPLETE' file. + ansible.builtin.file: + path: "{{ hpl_root }}/tmp/COMPILE_BLIS_COMPLETE" + state: touch + mode: 0644 diff --git a/tasks/algebra_openblas.yml b/tasks/algebra_openblas.yml new file mode 100644 index 0000000..ff17862 --- /dev/null +++ b/tasks/algebra_openblas.yml @@ -0,0 +1,22 @@ +--- +- name: Download OpenBLAS linear algebra library. + ansible.builtin.git: + repo: 'https://github.com/xianyi/OpenBLAS' + version: "{{ linear_algebra_openblas_version }}" + dest: "{{ hpl_root }}/tmp/openblas-build" + +- name: Install OpenBLAS. + ansible.builtin.command: "{{ item }}" + args: + chdir: "{{ hpl_root }}/tmp/openblas-build" + creates: "{{ hpl_root }}/tmp/COMPILE_OPENBLAS_COMPLETE" + loop: + - make -j{{ ansible_processor_vcpus }} + - make PREFIX=/opt/openblas install + become: true + +- name: Create 'COMPILE_OPENBLAS_COMPLETE' file. + ansible.builtin.file: + path: "{{ hpl_root }}/tmp/COMPILE_OPENBLAS_COMPLETE" + state: touch + mode: 0644 diff --git a/templates/benchmark-Make.rpi.j2 b/templates/benchmark-Make.top500.j2 similarity index 95% rename from templates/benchmark-Make.rpi.j2 rename to templates/benchmark-Make.top500.j2 index c60c8a9..6f5c368 100644 --- a/templates/benchmark-Make.rpi.j2 +++ b/templates/benchmark-Make.top500.j2 @@ -61,7 +61,7 @@ TOUCH = touch # - Platform identifier ------------------------------------------------ # ---------------------------------------------------------------------- # -ARCH = rpi +ARCH = top500 # # ---------------------------------------------------------------------- # - HPL Directory Structure / HPL library ------------------------------ @@ -92,9 +92,19 @@ MPlib = /usr/local/lib/libmpich.so # header files, LAlib is defined to be the name of the library to be # used. The variable LAdir is only used for defining LAinc and LAlib. # +{% if linear_algebra_library == 'atlas' %} LAdir = {{ hpl_root }}/tmp/atlas-build LAinc = LAlib = $(LAdir)/lib/libf77blas.a $(LAdir)/lib/libatlas.a +{% elif linear_algebra_library == 'blis' %} +LAdir = /opt/blis +LAinc = +LAlib = $(LAdir)/lib/libblis.a -lpthread +{% elif linear_algebra_library == 'openblas' %} +LAdir = /opt/openblas +LAinc = +LAlib = $(LAdir)/lib/libopenblas.a -lpthread +{% endif %} # # ---------------------------------------------------------------------- # - F77 / C interface --------------------------------------------------