Skip to content

Commit

Permalink
Merge pull request #1276 from Zarquan/20240206-zrq-handover-notes
Browse files Browse the repository at this point in the history 
20240206 zrq handover notes
  • Loading branch information
@Zarquan
Zarquan authored Feb 6, 2024
2 parents 45a8ada + d456677 commit eb76edb
Show file tree
Hide file tree
Showing 4 changed files with 336 additions and 2 deletions.
2 changes: 1 addition & 1 deletion deployments/admin/bin/create-user-tools.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@
#
# Call our Openstack script to create the share.
# Returns JSON.
/deployments/zeppelin/bin/create-ceph-share.sh \
/deployments/admin/bin/create-ceph-share.sh \
"${sharecloud}" \
"${sharename}" \
"${mountpath}" \
Expand Down
1 change: 1 addition & 0 deletions notes/zrq/20240204-02-test-deploy.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
Result:

Work in progress ...
Ran out of time, had to stop it.

# -----------------------------------------------------
# Check the current live host.
Expand Down
14 changes: 13 additions & 1 deletion notes/zrq/20240205-02-test-deploy.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@

Result:

Work in progress ...
Fail - network issues getting test code from GitHub.

Stelios tried this and this part worked OK.
He found issues later on in the deploy process allocating user shares.
Suspect network issues at Cambridge.



Expand Down Expand Up @@ -265,3 +269,11 @@ EOF
> Resolving deltas: 100% (11/11), done.


#
# Stelios tried this and this part worked OK.
# He found issues later on in the deploy process allocating user shares.
# Suspect networkissues at Cambridge.
#



321 changes: 321 additions & 0 deletions notes/zrq/20240206-01-handover-notes.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,321 @@
#
# <meta:header>
# <meta:licence>
# Copyright (c) 2024, ROE (http://www.roe.ac.uk/)
#
# This information is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This information is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# </meta:licence>
# </meta:header>
#
#zrq-notes-time
#zrq-notes-indent
#zrq-notes-crypto
#zrq-notes-ansible
#zrq-notes-osformat
#zrq-notes-zeppelin
#

Target:

Success

Result:

Work in progress ...


# -----------------------------------------------------
# Side note : DNS alias for live system.
#[user@laptop]

host dmp.gaia.ac.uk

> dmp.gaia.ac.uk is an alias for aglais-live.duckdns.org.
> aglais-live.duckdns.org has address 128.232.227.27


# -----------------------------------------------------
#

Login to Openstack Horizon at Cambridge (keystone auth)
https://arcus.openstack.hpc.cam.ac.uk/

Generate application credentials for each cloud (red, green, blue and data).
Fill in the appropriate sections in the clouds.yaml file on your desktop/laptop/vm.


# -----------------------------------------------------
#

Login to DigitalOcean and create a VM to use as the client.
Add public ssh key to the VM.


# -----------------------------------------------------
# Login to the VM using ssh.
#[user@windows]

ssh [email protected]


# -----------------------------------------------------
# Create our clouds.yaml credentials file.
# Use this as a template and fill in with your credentials.
#[user@client-vm]

vi "${HOME}/clouds.yaml"

clouds:

iris-gaia-red:
auth:
auth_url: https://arcus.openstack.hpc.cam.ac.uk:5000
application_credential_id: "######## #######"
application_credential_secret: "######## #######"
region_name: "RegionOne"
interface: "public"
identity_api_version: 3
auth_type: "v3applicationcredential"

iris-gaia-green:
auth:
auth_url: https://arcus.openstack.hpc.cam.ac.uk:5000
application_credential_id: "######## #######"
application_credential_secret: "######## #######"
region_name: "RegionOne"
interface: "public"
identity_api_version: 3
auth_type: "v3applicationcredential"

iris-gaia-blue:
auth:
auth_url: https://arcus.openstack.hpc.cam.ac.uk:5000
application_credential_id: "######## #######"
application_credential_secret: "######## #######"
region_name: "RegionOne"
interface: "public"
identity_api_version: 3
auth_type: "v3applicationcredential"

iris-gaia-data:
auth:
auth_url: https://arcus.openstack.hpc.cam.ac.uk:5000
application_credential_id: "######## #######"
application_credential_secret: "######## #######"
region_name: "RegionOne"
interface: "public"
identity_api_version: 3
auth_type: "v3applicationcredential"

somerville-gaia-jade:
auth:
auth_url: https://somerville.ed.ac.uk:5000
application_credential_id: "######## #######"
application_credential_secret: "######## #######"
region_name: "RegionOne"
interface: "public"
identity_api_version: 3
auth_type: "v3applicationcredential"


# -----------------------------------------------------
# Install client software
#[user@client-vm]

apt-get install -y podman

> ....
> ....


apt-get install -y git

> ....
> ....


# -----------------------------------------------------
# Create a local env file in your home directory.
# This makes things easier to refer to later on.
#[user@client-vm]

# This example uses the main wfau repo.
# You should use your own fork once you have one.

vi "${HOME}/aglais.env"

AGLAIS_REPO='https://github.com/wfau/gaia-dmp.git'
AGLAIS_CODE="${HOME}/gaia-dmp"

PATH="${PATH}:${AGLAIS_CODE}/bin"
export PATH


# -----------------------------------------------------
# Clone our main source code.
#[user@client-vm]

# Use this once you have your ssh keys setup on GitHub.
git clone '[email protected]:wfau/gaia-dmp.git'

# For editing you should use your fork rather than the main repo.
git clone '[email protected]:millingw/gaia-dmp.git'

# Use this form for a simple read-only copy.
git clone 'https://github.com/wfau/gaia-dmp.git'

> ....
> ....

# -----------------------------------------------------
# Setup your private ssh key.
#[user@client-vm]

# This is not needed if you are using a Linux client.
# Linux automatically runs a ssh agent that will respond to authentication requests.
# This doesn't work on Windows, so we need to start an agent on the client VM.

vi "${HOME}/.ssh/id_rsa"

-----BEGIN OPENSSH PRIVATE KEY-----
....
....
-----END OPENSSH PRIVATE KEY-----


# -----------------------------------------------------
# Setup your ssh-agent.
#[user@client-vm]

# This is not needed if you are using a Linux client.
# Linux automatically runs a ssh agent that will respond to authentication requests.
# This doesn't work on Windows, so we need to start an agent on the client VM.

eval $(ssh-agent)

> Agent pid 7003


cat << EOF
SSH_AGENT_PID [${SSH_AGENT_PID}]
SSH_AUTH_SOCK [${SSH_AUTH_SOCK}]
EOF

> SSH_AGENT_PID [7003]
> SSH_AUTH_SOCK [/tmp/ssh-AQGTppVhW5Ka/agent.7002]


# -----------------------------------------------------
# Add your ssh key to the agent.
#[user@client-vm]

# This is not needed if you are using a Linux client.
# Linux automatically runs a ssh agent that will respond to authentication requests.
# This doesn't work on Windows, so we need to start an agent on the client VM.

ssh-add "${HOME}/.ssh/id_rsa"

> Identity added: ........


ssh-add -l

> 3072 ........


# -----------------------------------------------------
# Sanity check the current live host.
#[user@client-vm]

ssh [email protected] \
'
date
hostname
'

> Tue Feb 6 20:38:34 UTC 2024
> iris-gaia-green-20231027-zeppelin


# -----------------------------------------------------
# Create a container to work with.
#[user@client-vm]

source "${HOME:?}/aglais.env"

ansi-client 'blue'

> ---- ---- ----
> File [ansi-client]
> Path [/root/gaia-dmp/bin]
> ---- ---- ----
> Cloud name [iris-gaia-blue]
> Cloud site [cambridge-arcus]
> Client name [ansibler-blue]
> Config name [zeppelin-54.86-spark-6.26.43]
> ---- ---- ----
> ....
> ....


# -----------------------------------------------------
# Deploy everything.
#[root@ansibler]

source /deployments/hadoop-yarn/bin/deploy.sh

> ....
> ....
> /
> aglais:
> status:
> deployment:
> type: hadoop-yarn
> conf: zeppelin-54.86-spark-6.26.43
> name: iris-gaia-blue-20240206
> date: 20240206T213631
> hostname: zeppelin.gaia-dmp.uk
> spec:
> openstack:
> cloud:
> base: arcus
> name: iris-gaia-blue


# -----------------------------------------------------
# Import our live users.
#[root@ansibler]

#
# This command is desgined to be repeatable.
# It will check for missing elements and create them.
# Which means you can run it multiple times to add new users and it won't create duplicates.
#

source /deployments/admin/bin/create-user-tools.sh

import-live-users

> ....
> ....
> "FAIL: Failed to create share [iris-gaia-data-user-SGoughKelly] return code [1]",
> "ShareSizeExceedsAvailableQuota: Requested share exceeds allowed project/user or share type gigabytes quota. (HTTP 413) (Request-ID: req-8ac0a3bc-7970-464d-a027-ebb793fcfe7a)",
> "SKIP: Mounting share [iris-gaia-data-user-SGoughKelly][] skipped, status []"

#
# This was when I was using Malcolm's Openstack credentials.
# I'll try again with my credentials.
#


0 comments on commit eb76edb

Please sign in to comment.