From f1295485fab6307bf7436e41032fb7fd96ada48b Mon Sep 17 00:00:00 2001 From: Kari Harju Date: Thu, 8 Jan 2026 14:24:00 +0200 Subject: [PATCH 1/2] Added tooling and guide for actions manifest regeneration --- bin/publisher/regenerate_manifests.py | 23 ++++---- bin/publisher/s3/README.md | 55 +++++++++++++++++-- .../s3/upload_regen_action_manifests.sh | 52 ++++++++++++++++++ 3 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 bin/publisher/s3/upload_regen_action_manifests.sh diff --git a/bin/publisher/regenerate_manifests.py b/bin/publisher/regenerate_manifests.py index 27a07bb0..57baeb38 100644 --- a/bin/publisher/regenerate_manifests.py +++ b/bin/publisher/regenerate_manifests.py @@ -110,6 +110,9 @@ def regenerate_manifests(): print("Starting manifest regeneration from local S3 content...") + # Check for SKIP_CLEAN environment variable + skip_clean = os.environ.get("SKIP_CLEAN", "").lower() in ("1", "true", "yes") + # Check if S3 content exists if not os.path.exists(s3_actions_folder): print(f"Error: S3 actions folder not found at {s3_actions_folder}") @@ -118,15 +121,15 @@ def regenerate_manifests(): ) return - # Clear existing folders (but keep temp-gallery if it exists and has content) + # Clear existing folders clear_folders(zips_folder) clear_folders(gallery_actions_folder) - # Only clear temp folder if it's empty or doesn't exist - if ( - not os.path.exists(temp_gallery_folder) - or len(os.listdir(temp_gallery_folder)) == 0 - ): + # Clear temp folder by default (use SKIP_CLEAN=1 to preserve cache) + if skip_clean and os.path.exists(temp_gallery_folder) and len(os.listdir(temp_gallery_folder)) > 0: + print("SKIP_CLEAN enabled: Using existing extracted packages from temp folder...") + else: + print("Clearing temp-extracts cache...") clear_folders(temp_gallery_folder) # Load whitelist first to know which packages to process @@ -137,10 +140,8 @@ def regenerate_manifests(): all_whitelisted_packages = set(whitelist["standard"] + whitelist["spcs"]) print(f"Processing {len(all_whitelisted_packages)} whitelisted packages") - # Check if temp folder already has extracted content - if os.path.exists(temp_gallery_folder) and len(os.listdir(temp_gallery_folder)) > 0: - print("Using existing extracted packages from temp folder...") - else: + # Extract packages if temp folder is empty + if not os.path.exists(temp_gallery_folder) or len(os.listdir(temp_gallery_folder)) == 0: # Copy individual zip files from S3 content to zips folder (only whitelisted packages) print("Copying zip files from S3 content (whitelisted packages only)...") copied_count = 0 @@ -232,8 +233,6 @@ def regenerate_manifests(): else: print(f"{package_name}: NOT FOUND in manifest") - # Keep temporary folder for future runs (since S3 packages are immutable) - print("Keeping extracted packages for future runs...") if __name__ == "__main__": diff --git a/bin/publisher/s3/README.md b/bin/publisher/s3/README.md index ca9193d5..b6e6c4be 100644 --- a/bin/publisher/s3/README.md +++ b/bin/publisher/s3/README.md @@ -1,7 +1,50 @@ -- Configure AWS profiles: - - `C:\Users\\.aws` +# S3 Actions Manifest Regeneration (Admin Only) -aws configure list-profiles -set AWS_PROFILE=cicd -aws sso login --profile cicd -aws s3 cp s3://downloads.robocorp.com/gallery/actions/ ./s3-actions/ --recursive \ No newline at end of file +If for whatever reason we run into a case where the action pack manifests are broken or missing versions, with this folder and guide we are able to regenerate the manifest based on the S3 content. The Action Packs and old versions are all there, we need to "just": +- Load the S3 content locally +- Regenerate the manifest files +- Push the updates back to S3 + +**Requires:** +- Git Bash on Windows +- AWS cli setup on the machine +- AWS CICD account with `ProductionAccountAdmin` role. + +`~/.aws/config` should have: +``` +[profile cicd-admin] +region = eu-west-1 +sso_start_url = https://d-9067b8f409.awsapps.com/start/# +sso_region = us-east-1 +sso_account_id = 710450854638 +sso_role_name = ProductionAccountAdmin +output = json +``` + +Login the aws cli: +``` +export AWS_PROFILE=cicd-admin +aws sso login --profile cicd-admin +``` + +## Steps + +Run the commands in the `/publisher` -root folder + +1. Download S3 content: + ``` + aws s3 cp s3://downloads.robocorp.com/gallery/actions/ ./s3/s3-actions/ --recursive + ``` + +2. Regenerate manifests: + ``` + rcc run -t "Regen actions manifests" + ``` + Use `SKIP_CLEAN=1` to skip cache clearing for faster runs if whitelist unchanged. + +3. Upload to S3: + ``` + ./s3/upload_regen_action_manifests.sh + ``` + +4. Verify the manifests in AWS S3 or that action packs show up in Studio diff --git a/bin/publisher/s3/upload_regen_action_manifests.sh b/bin/publisher/s3/upload_regen_action_manifests.sh new file mode 100644 index 00000000..55df6858 --- /dev/null +++ b/bin/publisher/s3/upload_regen_action_manifests.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -e + +# NOTES: THIS SCRIPT IS ONLY MEANT FOR PUSHING REGENERATED ACTION MANIFEST FILES TO S3 +# 1. Requires AWS CICD account admin permissions +# 2. You must follow the README.md (the regen part can be done with AWS 'ReadOnlyAccess' -role) +# 3. Run: rcc run -t "Regen actions manifests" +# 4. Switch to admin role in AWS +# 5. Run this script +# 6. Verify the manifests via AWS S3 or in Studio + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GALLERY_DIR="$SCRIPT_DIR/../gallery" +S3_BASE_URL="s3://downloads.robocorp.com/gallery/actions" + +# Check if AWS CLI is installed +if ! command -v aws &> /dev/null; then + echo "ERROR: AWS CLI is not installed." + exit 1 +fi + +# Check if AWS credentials are configured +if ! aws sts get-caller-identity &> /dev/null; then + echo "ERROR: AWS CLI is not configured or credentials have expired." + echo "" + echo "To configure AWS CLI:" + echo " 1. You need to have 'ProductionAccountAdmin' role in AWS" + echo " 2. Ensure AWS profile is configured in ~/.aws/" + echo " 3. Run: export AWS_PROFILE=cicd-admin" + echo " 4. Run: aws sso login --profile cicd-admin" + echo "" + echo "See s3/README.md for more details." + exit 1 +fi + +echo "AWS CLI configured. Using identity:" +aws sts get-caller-identity --query 'Arn' --output text +echo "" + +cd "$GALLERY_DIR" + +echo "Uploading manifest.json..." +aws s3 cp manifest.json $S3_BASE_URL/manifest.json --cache-control max-age=120 --content-type "application/json" +sha256sum manifest.json | awk '{printf "%s", $1}' > manifest.sha256 +aws s3 cp manifest.sha256 $S3_BASE_URL/manifest.sha256 --cache-control max-age=120 --content-type "text/plain" + +echo "Uploading manifest_spcs.json..." +aws s3 cp manifest_spcs.json $S3_BASE_URL/manifest_spcs.json --cache-control max-age=120 --content-type "application/json" +sha256sum manifest_spcs.json | awk '{printf "%s", $1}' > manifest_spcs.sha256 +aws s3 cp manifest_spcs.sha256 $S3_BASE_URL/manifest_spcs.sha256 --cache-control max-age=120 --content-type "text/plain" + +echo "Done!" From 16df18893716e7ff42933a6905e97b990a07fc58 Mon Sep 17 00:00:00 2001 From: Kari Harju Date: Thu, 8 Jan 2026 14:25:37 +0200 Subject: [PATCH 2/2] Update README.md --- bin/publisher/s3/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/publisher/s3/README.md b/bin/publisher/s3/README.md index b6e6c4be..fda36373 100644 --- a/bin/publisher/s3/README.md +++ b/bin/publisher/s3/README.md @@ -16,8 +16,8 @@ If for whatever reason we run into a case where the action pack manifests are br region = eu-west-1 sso_start_url = https://d-9067b8f409.awsapps.com/start/# sso_region = us-east-1 -sso_account_id = 710450854638 -sso_role_name = ProductionAccountAdmin +sso_account_id = +sso_role_name = output = json ```