Action regen tooling

bin/publisher/regenerate_manifests.py

@@ -110,6 +110,9 @@ def regenerate_manifests():
 
     print("Starting manifest regeneration from local S3 content...")
 
+    # Check for SKIP_CLEAN environment variable
+    skip_clean = os.environ.get("SKIP_CLEAN", "").lower() in ("1", "true", "yes")
+
     # Check if S3 content exists
     if not os.path.exists(s3_actions_folder):
         print(f"Error: S3 actions folder not found at {s3_actions_folder}")
@@ -118,15 +121,15 @@ def regenerate_manifests():
         )
         return
 
-    # Clear existing folders (but keep temp-gallery if it exists and has content)
+    # Clear existing folders
     clear_folders(zips_folder)
     clear_folders(gallery_actions_folder)
 
-    # Only clear temp folder if it's empty or doesn't exist
-    if (
-        not os.path.exists(temp_gallery_folder)
-        or len(os.listdir(temp_gallery_folder)) == 0
-    ):
+    # Clear temp folder by default (use SKIP_CLEAN=1 to preserve cache)
+    if skip_clean and os.path.exists(temp_gallery_folder) and len(os.listdir(temp_gallery_folder)) > 0:
+        print("SKIP_CLEAN enabled: Using existing extracted packages from temp folder...")
+    else:
+        print("Clearing temp-extracts cache...")
         clear_folders(temp_gallery_folder)
 
     # Load whitelist first to know which packages to process
@@ -137,10 +140,8 @@ def regenerate_manifests():
     all_whitelisted_packages = set(whitelist["standard"] + whitelist["spcs"])
     print(f"Processing {len(all_whitelisted_packages)} whitelisted packages")
 
-    # Check if temp folder already has extracted content
-    if os.path.exists(temp_gallery_folder) and len(os.listdir(temp_gallery_folder)) > 0:
-        print("Using existing extracted packages from temp folder...")
-    else:
+    # Extract packages if temp folder is empty
+    if not os.path.exists(temp_gallery_folder) or len(os.listdir(temp_gallery_folder)) == 0:
         # Copy individual zip files from S3 content to zips folder (only whitelisted packages)
         print("Copying zip files from S3 content (whitelisted packages only)...")
         copied_count = 0
@@ -232,8 +233,6 @@ def regenerate_manifests():
         else:
             print(f"{package_name}: NOT FOUND in manifest")
 
-    # Keep temporary folder for future runs (since S3 packages are immutable)
-    print("Keeping extracted packages for future runs...")
 
 
 if __name__ == "__main__":

bin/publisher/s3/README.md

@@ -1,7 +1,50 @@
-- Configure AWS profiles:
-  - `C:\Users\<user>\.aws`
+# S3 Actions Manifest Regeneration (Admin Only)
 
-aws configure list-profiles
-set AWS_PROFILE=cicd
-aws sso login --profile cicd
-aws s3 cp s3://downloads.robocorp.com/gallery/actions/ ./s3-actions/ --recursive
+If for whatever reason we run into a case where the action pack manifests are broken or missing versions, with this folder and guide we are able to regenerate the manifest based on the S3 content. The Action Packs and old versions are all there, we need to "just":
+- Load the S3 content locally
+- Regenerate the manifest files
+- Push the updates back to S3
+
+**Requires:** 
+- Git Bash on Windows
+- AWS cli setup on the machine
+- AWS CICD account with `ProductionAccountAdmin` role.
+
+`~/.aws/config` should have:
+```
+[profile cicd-admin]
+region = eu-west-1
+sso_start_url = https://d-9067b8f409.awsapps.com/start/#
+sso_region = us-east-1
+sso_account_id = <account id from AWS portal>
+sso_role_name = <role name from AWS portal>
+output = json
+```
+
+Login the aws cli:
+```
+export AWS_PROFILE=cicd-admin
+aws sso login --profile cicd-admin
+```
+
+## Steps
+
+Run the commands in the `/publisher` -root folder
+
+1. Download S3 content:
+   ```
+   aws s3 cp s3://downloads.robocorp.com/gallery/actions/ ./s3/s3-actions/ --recursive
+   ```
+
+2. Regenerate manifests:
+   ```
+   rcc run -t "Regen actions manifests"
+   ```
+   Use `SKIP_CLEAN=1` to skip cache clearing for faster runs if whitelist unchanged.
+
+3. Upload to S3:
+   ```
+   ./s3/upload_regen_action_manifests.sh
+   ```
+
+4. Verify the manifests in AWS S3 or that action packs show up in Studio

bin/publisher/s3/upload_regen_action_manifests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -e
+
+# NOTES: THIS SCRIPT IS ONLY MEANT FOR PUSHING REGENERATED ACTION MANIFEST FILES TO S3
+# 1. Requires AWS CICD account admin permissions
+# 2. You must follow the README.md (the regen part can be done with AWS 'ReadOnlyAccess' -role)
+# 3. Run: rcc run -t "Regen actions manifests"
+# 4. Switch to admin role in AWS
+# 5. Run this script
+# 6. Verify the manifests via AWS S3 or in Studio
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+GALLERY_DIR="$SCRIPT_DIR/../gallery"
+S3_BASE_URL="s3://downloads.robocorp.com/gallery/actions"
+
+# Check if AWS CLI is installed
+if ! command -v aws &> /dev/null; then
+    echo "ERROR: AWS CLI is not installed."
+    exit 1
+fi
+
+# Check if AWS credentials are configured
+if ! aws sts get-caller-identity &> /dev/null; then
+    echo "ERROR: AWS CLI is not configured or credentials have expired."
+    echo ""
+    echo "To configure AWS CLI:"
+    echo "  1. You need to have 'ProductionAccountAdmin' role in AWS"
+    echo "  2. Ensure AWS profile is configured in ~/.aws/"
+    echo "  3. Run: export AWS_PROFILE=cicd-admin"
+    echo "  4. Run: aws sso login --profile cicd-admin"
+    echo ""
+    echo "See s3/README.md for more details."
+    exit 1
+fi
+
+echo "AWS CLI configured. Using identity:"
+aws sts get-caller-identity --query 'Arn' --output text
+echo ""
+
+cd "$GALLERY_DIR"
+
+echo "Uploading manifest.json..."
+aws s3 cp manifest.json $S3_BASE_URL/manifest.json --cache-control max-age=120 --content-type "application/json"
+sha256sum manifest.json | awk '{printf "%s", $1}' > manifest.sha256
+aws s3 cp manifest.sha256 $S3_BASE_URL/manifest.sha256 --cache-control max-age=120 --content-type "text/plain"
+
+echo "Uploading manifest_spcs.json..."
+aws s3 cp manifest_spcs.json $S3_BASE_URL/manifest_spcs.json --cache-control max-age=120 --content-type "application/json"
+sha256sum manifest_spcs.json | awk '{printf "%s", $1}' > manifest_spcs.sha256
+aws s3 cp manifest_spcs.sha256 $S3_BASE_URL/manifest_spcs.sha256 --cache-control max-age=120 --content-type "text/plain"
+
+echo "Done!"