Merge pull request #160 from bedroge/eessi.io

Switch to new infrastructure + eessi.io repo

Author: boegel

.github/workflows/build-test-release-client-packages.yml

@@ -147,12 +147,12 @@ jobs:
         run: sudo cvmfs_config setup
 
       - name: Test repository access
-        run: ls /cvmfs/pilot.eessi-hpc.org/
+        run: ls /cvmfs/software.eessi.io/
 
   test-rpm-package:
     needs: build-linux-packages
     runs-on: ubuntu-latest
-    container: 
+    container:
       image: centos:centos7
       options: --device /dev/fuse --privileged
 
@@ -183,10 +183,10 @@ jobs:
 
       - name: Mount the repositories
         #run: cvmfs_config setup
-        run: mkdir -p /cvmfs/pilot.eessi-hpc.org && mount -t cvmfs pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org
+        run: mkdir -p /cvmfs/software.eessi.io && mount -t cvmfs software.eessi.io /cvmfs/software.eessi.io
 
       - name: Test repository access
-        run: ls /cvmfs/pilot.eessi-hpc.org/
+        run: ls /cvmfs/software.eessi.io/
 
   test-tar-package:
     needs: build-linux-packages
@@ -223,7 +223,7 @@ jobs:
         run: sudo cvmfs_config setup
 
       - name: Test repository access
-        run: ls /cvmfs/pilot.eessi-hpc.org/
+        run: ls /cvmfs/software.eessi.io/
 
 #  test-macos-package:
 #    needs: build-macos-package
@@ -254,10 +254,10 @@ jobs:
 #        run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local
 #
 #      - name: Mount the repositories
-#        run: sudo mkdir -p /Users/Shared/cvmfs/pilot.eessi-hpc.org && sudo mount -t cvmfs pilot.eessi-hpc.org /Users/Shared/cvmfs/pilot.eessi-hpc.org
+#        run: sudo mkdir -p /Users/Shared/cvmfs/software.eessi.io && sudo mount -t cvmfs software.eessi.io /Users/Shared/cvmfs/software.eessi.io
 #
 #      - name: Test repository access
-#        run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/
+#        run: ls /Users/Shared/cvmfs/software.eessi.io/
 
   release:
     #needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package, test-tar-package]

inventory/group_vars/all.yml

@@ -8,7 +8,7 @@ eessi_cvmfs_repos_enabled: true
 eessi_email: support@eessi.io
 
 #
-# Defaults for eessi-hpc.org repos.
+# Defaults for eessi.io repos.
 #
 
 # We do not use a config repo anymore.
@@ -18,97 +18,39 @@ eessi_cvmfs_config_repo: {}
 # /etc/cvmfs/keys/*.pub
 # Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys.
 eessi_cvmfs_keys:
-  - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub
+  - path: /etc/cvmfs/keys/eessi.io/software.eessi.io.pub
     key: |
-     -----BEGIN PUBLIC KEY-----
-     MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqoTBc/Eksr9ECEOoCzHA
-     F20UjePR7mJyWB13eTBXnjq7PLKKilfk2llLKetPIRMEmbvp97NiQ9ALe/tiGDzT
-     1Wg82Mf9oO4DNQYrTfE/MP5inIrMtSv1WnYJYV4Bgm24s8IhHXytIf1D4qt0x1v5
-     0yRui0dwaZrErWKRKufbJ6ehmEDHuVSEAujrORyig/jYFKVyTaCsXnmwFDJmX2Uw
-     RGCaf0M9ZLYs9t/SWA0s08rle5kH82hgBsIvUr7PLT7hbU407VK6+Y02tuxbI0JR
-     +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk
-     BwIDAQAB
-     -----END PUBLIC KEY-----
-  - path: /etc/cvmfs/keys/eessi-hpc.org/pilot.eessi-hpc.org.pub
-    key: |
-     -----BEGIN PUBLIC KEY-----
-     MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg
-     8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8
-     v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo
-     4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ
-     uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm
-     HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby
-     FQIDAQAB
-     -----END PUBLIC KEY-----
-  # The following (duplicate) key can be removed after this PR has been merged:
-  # https://github.com/galaxyproject/ansible-cvmfs/pull/32
-  - path: /etc/cvmfs/keys/eessi-hpc.org/data.eessi-hpc.org.pub
-    key: |
-     -----BEGIN PUBLIC KEY-----
-     MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg
-     8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8
-     v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo
-     4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ
-     uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm
-     HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby
-     FQIDAQAB
-     -----END PUBLIC KEY-----
-  - path: /etc/cvmfs/keys/eessi-hpc.org/ci.eessi-hpc.org.pub
-    key: |
-     -----BEGIN PUBLIC KEY-----
-     MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg
-     8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8
-     v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo
-     4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ
-     uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm
-     HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby
-     FQIDAQAB
-     -----END PUBLIC KEY-----
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyau1UFUcoiqpE5U9StON
+      W0Trc3PM02AA5kYgknrqZJdusj5PcNg7rhOnrd+SX8BIiVtVMr6mqORXsJ2FNydJ
+      lfm512GcmurM154m6Hz4I+UttcpVLe1CvlyxjjCrN0siSiPnLDONy3Ih1PWyRlA/
+      sobSaIhx3Vyn1IzC0nbjstKObAYJs2kjokS+cipX2oIXJSoxkhvXcxmiMCi2GQIs
+      KTQDT+bATs9eK+ZqY4n5x8VX87AlMBQuSCP5mA8VQPx4aCf8AjHWAn3qmPrhk3od
+      Z/Cl9vrYzoWJT1HOyumv7aEv7UlY81++3w/AwYKXLJI6ioop2AotjrUm3w2U3+k7
+      WQIDAQAB
+      -----END PUBLIC KEY-----
 
 # URLs for all the Stratum 1 servers.
 eessi_cvmfs_server_urls:
-  - domain: eessi-hpc.org
+  - domain: eessi.io
     use_geoapi: true
     urls:
-      - "http://rug-nl.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@"
-      - "http://bgo-no.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@"
-      - "http://azure-us-east1.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@"
-      - "http://aws-eu-west1.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@"
+      - "http://aws-eu-central-s1.eessi.science/cvmfs/@fqrn@"
+      - "http://azure-us-east-s1.eessi.science/cvmfs/@fqrn@"
 
 # Configuration of all the repositories.
 eessi_cvmfs_repositories:
-  - repository: pilot.eessi-hpc.org
-    stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org
+  - repository: software.eessi.io
+    stratum0: rug-nl-s0.eessi.science
     owner: "{{ cvmfs_repo_owner | default('root') }}"
-    key_dir: /etc/cvmfs/keys/eessi-hpc.org
+    key_dir: /etc/cvmfs/keys/eessi.io
     server_options:
       - CVMFS_AUTO_GC=false
       - CVMFS_AUTO_TAG_TIMESPAN="30 days ago"
       - CVMFS_GARBAGE_COLLECTION=true
     client_options: []
     use_for_ci: false
-    description: Repository containing pilot versions of the EESSI software stack.
-  - repository: data.eessi-hpc.org
-    stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org
-    owner: "{{ cvmfs_repo_owner | default('root') }}"
-    key_dir: /etc/cvmfs/keys/eessi-hpc.org
-    server_options:
-      - CVMFS_AUTO_GC=false
-      - CVMFS_AUTO_TAG_TIMESPAN="30 days ago"
-      - CVMFS_GARBAGE_COLLECTION=true
-    client_options: []
-    use_for_ci: false
-    description: "Data files used for applications available in the EESSI stack."
-  - repository: ci.eessi-hpc.org
-    stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org
-    owner: "{{ cvmfs_repo_owner | default('root') }}"
-    key_dir: /etc/cvmfs/keys/eessi-hpc.org
-    server_options:
-      - CVMFS_AUTO_GC=false
-      - CVMFS_GARBAGE_COLLECTION=true
-    client_options: []
-    use_for_ci: true
-    description: "Small-sized repository used for CI purposes."
+    description: "Repository that contains the EESSI software stacks."
 
 # Meta information for CVMFS Stratum servers
 cvmfs_server_meta_administrator: "EESSI CVMFS Administrators"

prepare-client-packages.yml

@@ -40,10 +40,10 @@
     - name: Make EESSI CVMFS domain configuration file
       copy:
         content: |
-          # Stratum 1 servers for the eessi-hpc.org domain
+          # Stratum 1 servers for the eessi.io domain
           CVMFS_SERVER_URL="{{ item.urls|join(';') }}"
 
-          # Public keys for the eessi-hpc.org domain
+          # Public keys for the eessi.io domain
           CVMFS_KEYS_DIR="/etc/cvmfs/keys/{{ item.domain }}"
 
           # Enable or disable the Geo API, based on the configuration file

roles/create_cvmfs_content_structure/files/.cvmfsdirtab

@@ -9,7 +9,7 @@
 /versions/*/software/*/*/*/*/software
 /versions/*/software/*/*/*/*/software/*/*
 /versions/*/software/*/*/*/*/modules
-# generic and graviton2 are one level less deep
+# generic and some (aarch64) targets are one level less deep
 /versions/*/software/*/*/*/software
 /versions/*/software/*/*/*/software/*/*
 /versions/*/software/*/*/*/modules

roles/create_cvmfs_content_structure/tasks/main.yml

@@ -5,17 +5,18 @@
   delegate_to: localhost
   ansible.builtin.stat:
     path: "{{ role_path }}/vars/{{ item.repository }}.yml"
-  with_items: "{{ eessi_cvmfs_repositories }}"
-  register: repo_vars_file
+  with_items: "{{ cvmfs_repositories | default(eessi_cvmfs_repositories) }}"
+  register: repo_vars_files
 
 - name: "Apply do_repo for each repository with a vars file"
   ansible.builtin.include_tasks: do_repo.yml
   vars:
     cvmfs_repo: "{{ repo_vars_file.item.repository }}"
   when: repo_vars_file.stat.exists
-  with_items: "{{ repo_vars_file.results }}"
+  with_items: "{{ repo_vars_files.results }}"
   loop_control:
     loop_var: repo_vars_file
   args:
     apply:
-      become: "{{ 'yes' if repo_vars_file.item.owner == 'root' else 'no' }}"
+      become: "{{ 'true' if repo_vars_file.item.owner != lookup('env', 'USER') else 'false' }}"
+      become_user: "{{ repo_vars_file.item.owner }}"

roles/create_cvmfs_content_structure/vars/software.eessi.io.yml

@@ -0,0 +1,10 @@
+# Specifications of files and symlinks for the software.eessi.io CVMFS repository.
+# Paths for files and symlinks should be relative to the root of the repository.
+---
+files:
+  - name: .cvmfsdirtab
+    dest: ''
+    mode: '644'
+
+symlinks:
+  host_injections: '$(EESSI_HOST_INJECTIONS:-/opt/eessi)'

roles/cvmfs_server_meta_info/tasks/main.yml

@@ -7,6 +7,7 @@
     group: root
     mode: 0644
   become: true
+  when: (cvmfs_repositories | length) > 0
 
 - name: Create meta information for each CVMFS repository
   ansible.builtin.include_tasks: repo_meta_info.yml
@@ -17,6 +18,6 @@
       recommended-stratum0: "http://{{ item.stratum0 }}/cvmfs/{{ item.repository }}"
       recommended-stratum1s: "{{ eessi_cvmfs_server_urls | selectattr('domain', 'in', item.repository) |
                                  map(attribute='urls') | map('regex_replace', '@fqrn@', item.repository) }}"
-  with_items: "{{ eessi_cvmfs_repositories }}"
+  with_items: "{{ cvmfs_repositories | default(eessi_cvmfs_repositories) }}"
   when: "'cvmfsstratum0servers' in group_names or cvmfs_role == 'stratum0'"
 ...

roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml

@@ -18,15 +18,15 @@
 
 - name: Get the current meta information of this repository from the server
   ansible.builtin.command:
-    cmd: "cvmfs_swissknife info -M -r {{ this_cvmfs_repo_meta['recommended-stratum0'] }}"
+    cmd: "cvmfs_swissknife info -M -r http://localhost:{{ cvmfs_stratum0_http_ports[0] | default('80') }}/cvmfs/{{ this_cvmfs_repo.repository }}"
   changed_when: false
   register: current_repo_meta
 
 - name: Update the repository's meta information
   ansible.builtin.command:
     cmd: "cvmfs_server update-repoinfo -f {{ tmp_json_file.path }} {{ this_cvmfs_repo.repository }}"
   when: (current_repo_meta.stdout | checksum) != json_file_stat.stat.checksum
-  become_user: "{{ cvmfs_repo_owner }}"
+  become_user: "{{ cvmfs_repo_owner | default('root') }}"
 
 - name: Remove temporary json file
   ansible.builtin.file:

scripts/ingest-tarball.sh

@@ -12,7 +12,7 @@
 
 # Only if it passes these checks, the tarball gets ingested to the base dir in the repository specified below.
 
-repo=pilot.eessi-hpc.org
+repo=software.eessi.io
 basedir=versions
 decompress="gunzip -c"
 # list of supported architectures for compat and software layers

stratum0.yml

@@ -17,4 +17,5 @@
       when: ansible_facts['os_family'] == 'RedHat'
     - galaxyproject.cvmfs
     - cvmfs_server_meta_info
+    - create_cvmfs_content_structure
 ...