Merge pull request #1240 from akrause2014/fedora-38

Fedora 38 and Spark 3.2.4

deployments/common/pip/requirements.txt

@@ -1,21 +1,20 @@
-numpy==1.21.0
-scipy==1.7.3
-matplotlib==3.4.2
-grpcio==1.53.0
+numpy==1.26.2
+scipy==1.11.4
+matplotlib==3.8.2
+grpcio==1.59.3
 jupyter==1.0.0
-Cython==0.29.23
-protobuf==3.18.3
-pandas==1.2.4
-healpy==1.14.0
-astropy==4.2.1
-astroquery==0.4.1
-scikit-learn==0.24.2
-joblib==1.2.0
-hdbscan==0.8.31
+Cython==0.29.36
+protobuf==4.25.1
+pandas==2.1.3
+healpy==1.16.6
+astropy==5.3.4
+astroquery==0.4.6
+scikit-learn==1.3.2
+joblib==1.3.2
+hdbscan==0.8.33
 pyvo==1.1
-pyarrow==12.0.1
-koalas==1.8.2
-GaiaXPy==1.1.4
+pyarrow==14.0.1
+GaiaXPy==2.1.0
 git+https://github.com/wfau/[email protected]
 git+https://github.com/stvoutsin/[email protected]
 dustmaps==1.0.12

deployments/common/ssh/aglais-team-keys

@@ -3,5 +3,4 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3T5ShHZ+HQJ6LpPwgpqRK/U0SYnGLSGY7LtwPiDA4
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDzZl6gXOSEAKBaxiIb+YMZNc7UZmp/ruB9h/iJL/8fO6Y60JXlpnHW9hYv5ks/NFteokK9+5YSnW5WEVHZtLkCWRGSkAea2JrC64OSngGf4YPzLosxC3wDjjWwYPEya8nfkiVSSxWUXSNIfrdExDTaPEKB/IbFcHI6oJnDGtLEk9rdMQI+BKm5JPlg/gbOEBflAdB107dB5TSii9bj833oW1PyCOc8Ti+7JrH9qsxGUdjHhSYYw/bluxsMB+YoxA85Xu2uoikAW3LIIuNKfJGzdACvxLH40qnfYwbDrf5d7VHfWmYLT7x7ycKJiMbzmfNJMPfeeFMMldY02P+ubzT [email protected]
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7i4yiwoibOPn21ACNFfXZHQN2j7TzyuLLyaSoH97dUJyr0ImH5D/sVwcjK9O1+zagGuDMZGpBuP5Vi5tHQtvXu0nzwtOX1edX+/B07oTwvKCio8d5awhF0OxeA+YpSRbSKorrckPgcmXIxB2STKZeFxmm8OGrTyIA1gER1j2cgSlz2c+/ZYaO+9sOEX4WhnSA1MfiGLyqPJRvGQRXNeS3NhMRkw7dyi6ge2ohFrAN3miOuhW+vEjyc9yncjqyyfhI8l6FuJmaCpezX0dSLsR+TJBd9/OmsdUZTWB0uIMYDUAKQ2m5sglHMFiZ92F9XBYI0fcwMY1dgRc/F0qLRSmbSTTT1cCb9WRWlzcPrl4f4tcsu2RWChN6jMNTFp7yQ72dhKCqPrlkalk870ZRvXMANtNzL6Q4imeSMYPWAEH14p2N4SPRFEsTrH5iW39fThUzOCPbf9DLs+9FNmgk3SQkYlsQJZA2f8QDpd0joSLvBZER1BEr6M3xqvFTj/LqH88= [email protected]
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJYb748Ma79ycA5C1Nt9FeqlU4idNE+1LnUrEGqFnkeLV5QFbRj5DRBev3xbulFmkV09ZsV58fKnATtLjTTpgYUjuAEGVdqmf9n69TJCeTkB06zYbPxOBmjkEDUEcQ7BYATvZfRm8aI3VWUvivdOrjmUSBjrPqkHOw+FhlfTbGKkLo0osycHtALZMst46RuUm7s/XfbBQAfAksK85yx4ni98LcTyJk698nwavDGDjWOk/hjNwcTqFs1xRVUZmBalPi85J0CFou59t2jP9eRF7c97obMwVvniLTT7Jnhe/cB/806oQKPZBa13m7yprn85vmm9Axje1e059bP8ZrG2PX3RTwsl42oyAtCIuvZQ3rkhCP2NFIWjbkCPKPOy6RKOWe7h7C8yViiXyAHro3PoyTTd+dfkp7NVRjcgBeamrVdpR8McUJRU72upN0lXurhlkD8Fvoo8ptbwjAOw/ud/+Z53pRr3j6sF5Zqa7znkMNvjM4/jbxdxyCH59Ff4nzGs8= [email protected]
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJFJirE/fseIB0oNhuh6aVOiOPdsNgJe6CuUhXm9jkuy3aayBhg4Uxv5+ffBiMwZknxHMuU5v7Pp8SWbxFwXR1J4KbghkwoEP+NLIWY4V5nyporW02SPNcAy06ydNAUGHHClAzqFlXi0tJlA2K170Z3YgyNffyMWGf+znl1vlwSkYJxJtgX5ursgPzxz1ojFMvTJTNUM5diiRVzaRLbpJ3ivu9JneepHrHc1l3cgCrYp/dRgaYAdRPt7b2XAHYGXcCdFLGdeiLdcs+OhBU4DPW3k6pxYaAhBQLqLjZJ9hOMUsao0R9fFBTtyDUFI7mO+bgJ0Nm1qZ2qf71QZ3al5LCh4O2MXry46S/FkyzUVcdhekNircDFwwgX97egrcOiF9cOg8Wo3o+4l/Ce9YmMSU6q32yOfCckQywRegxsAnzIgea1cgJlF6xrjprpdft6KP2AlFdJz22qa7ibOea6MIl4AT3NduWghu3nrpTsTIEs9TcbbBPO775ImSeBQLTKi0= [email protected]
-
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtSEEGQBqIpSSUxbQGaVgs/iThzfdRXJdxEYVT0T6xy1uUzxZb8ZG/ClMAWTxIK7KBzD3HgJWK3sQ3c/A557+9FAjhM/P3dfActH6hz0tw3TtcnDiOFoy/yDliOcZTppMfl6ghaQJMOabnTnYuOWnQUzVPexeEFmRAPnJxgWlL8NuNJjqUFSXGx5F4O4tsOWSaxZJawEqSHMAJcjciEqkg2px7jdKj6tlvQjXV+9DZly2IBpI6NIhVBM1fGEhIAt31fLeQbzaOStongyRXofkOclsrShjECr088ZaZHCpNtzgQ1rkxSaPnaP89IcJEJ2SN3PVWB9NP/Fn8FdIFYF1B akrause@eidf001-wfau

deployments/hadoop-yarn/ansible/28-install-zeppelin-requirements.yml

@@ -52,31 +52,3 @@
       dnf:
         name:  'gcc'
         state: present
-
-    # Why do we need this ?
-    - name: "Install Python3-devel package"
-      become: true
-      dnf:
-        name:  'python3-devel'
-        state: present
-
-    # This is already done in 23-install-python
-    - name: "Set Python alternative to Python3"
-      become: true
-      command: update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-
-    # This is already done in 23-install-python
-    - name: "Create [/etc/profile.d/python.sh]"
-      become: true
-      blockinfile:
-        dest:  '/etc/profile.d/python.sh'
-        state: present
-        owner: 'root'
-        group: 'root'
-        mode:  'u=rw,g=r,o=r'
-        create: true
-        insertafter: 'EOF'
-        marker: '# {mark} Ansible managed Python'
-        block: |
-          alias python=python3
-

deployments/hadoop-yarn/ansible/29-install-pip-libs.yml

@@ -51,22 +51,17 @@
         name:  'gcc'
         state: present
 
-    # Why do we need this ?
-    - name: "Install Python3-devel package"
-      become: true
-      dnf:
-        name:  'python3-devel'
-        state: present
-
     - name: "Install required system libraries"
       become: true
       dnf:
-        name: libtiff-devel,libjpeg-devel,libzip-devel,freetype-devel,lcms2-devel,libwebp-devel,tcl-devel,tk-devel,libffi,libffi-devel
+        name: git,libtiff-devel,libjpeg-devel,libzip-devel,freetype-devel,lcms2-devel,libwebp-devel,tcl-devel,tk-devel,libffi,libffi-devel
         state: present
 
-    - name: "Upgrade pip"
-      become: true
-      command: "pip install --upgrade pip"
+# Do not upgrade pip on Fedora: https://github.com/pypa/pip/issues/11631
+#
+#    - name: "Upgrade pip"
+#      become: true
+#      command: "pip install --upgrade pip"
 
     - name: Copy pip requirements file into tmp
       become: true

deployments/hadoop-yarn/ansible/38-install-user-db.yml

@@ -135,18 +135,18 @@
          database={{shirodbname}}
 
    - name: "Create MariaDB database [{{shirodbname}}]"
-     become: true
      mysql_db:
        name: "{{shirodbname}}"
        state: 'present'
+       login_unix_socket: /var/lib/mysql/mysql.sock
 
    - name: "Create MariaDB user [{{shirodbuser}}]"
-     become: true
      mysql_user:
        name: "{{shirodbuser}}"
        password: "{{shirodbpass}}"
        priv: "{{shirodbname}}.*:ALL"
        state: "present"
+       login_unix_socket: /var/lib/mysql/mysql.sock
 
    - name: "Create Shiro Configuration"
      copy:
@@ -166,6 +166,7 @@
 
    - name: "Create Shiro tables"
      mysql_query:
+        login_unix_socket: /var/lib/mysql/mysql.sock
         login_db: "{{shirodbname}}"
         query:
             - CREATE TABLE IF NOT EXISTS users (

deployments/hadoop-yarn/ansible/config/spark.yml

@@ -21,10 +21,9 @@
 #
 
 # Spark vars
-spname: "spark-3.1.2"
-spfull: "spark-3.1.2-bin-hadoop3.2"
+spname: "spark-3.2.4"
+spfull: "spark-3.2.4-bin-hadoop3.2"
 spbase: "/opt"
 sphome: "/opt/spark"
 sphost: "master01"
 spuser: "fedora"
-