pg_tde test PR

.cirrus.tasks.yml

@@ -126,13 +126,13 @@ task:
 
 
 task:
-  name: FreeBSD - 13 - Meson
+  name: FreeBSD - Meson
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
     TEST_JOBS: 8
-    IMAGE_FAMILY: pg-ci-freebsd-13
+    IMAGE_FAMILY: pg-ci-freebsd
     DISK_SIZE: 50
 
     CCACHE_DIR: /tmp/ccache_dir
@@ -777,14 +777,11 @@ task:
   # - Don't use ccache, the files are uncacheable, polluting ccache's
   #   cache
   # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
-  # - XXX have to disable ICU to avoid errors:
-  #   https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
   ###
   always:
     headers_headerscheck_script: |
       time ./configure \
         ${LINUX_CONFIGURE_FEATURES} \
-        --without-icu \
         --quiet \
         CC="gcc" CXX"=g++" CLANG="clang"
       make -s -j${BUILD_JOBS} clean

.git-blame-ignore-revs

@@ -14,6 +14,9 @@
 #
 # $ git log --pretty=format:"%H # %cd%n# %s" $PGINDENTGITHASH -1 --date=iso
 
+71da1f03f2f0ff18ed11e4ba6b07b6bd56705a5d # 2025-04-22 11:40:24 +0200
+# Run pgperltidy
+
 c739ae9e288c095cfe1b91ce27a2f2c075ed5da4 # 2024-08-26 16:16:09 -0700
 # Fix identation.
 

.github/CODEOWNERS

@@ -1,6 +1,5 @@
 # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
 # Order is important; the last matching pattern takes the most precedence.
 
-*                                 @dutow @dAdAbird
-/contrib/pg_tde/documentation/    @nastena1606
+/contrib/pg_tde/documentation/    @nastena1606 @Andriciuc
 /.github/                         @artemgavrilov

.github/codecov.yml

@@ -0,0 +1,45 @@
+codecov:
+  strict_yaml_branch: TDE_REL_17_STABLE
+fixes:
+  - "src/::contrib/pg_tde/src/"  # move path for codecov file mappings e.g., "src/" => "contrib/pg_tde/src/"
+coverage:
+  status:
+    project:
+      default:
+        target: 90%
+        threshold: 1%
+        base: auto
+comment:
+  layout: "header, diff, components"
+component_management:
+  default_rules:
+    statuses:
+      - type: project
+        target: auto
+        branches:
+          - "TDE_REL_17_STABLE"
+  individual_components:
+    - component_id: access
+      paths:
+        - contrib/pg_tde/src/access/**
+    - component_id: catalog
+      paths:
+        - contrib/pg_tde/src/catalog/**
+    - component_id: common
+      paths:
+        - contrib/pg_tde/src/common/**
+    - component_id: encryption
+      paths:
+        - contrib/pg_tde/src/encryption/**
+    - component_id: keyring
+      paths:
+        - contrib/pg_tde/src/keyring/**
+    - component_id: src
+      paths:
+        - contrib/pg_tde/src/*.c
+    - component_id: smgr
+      paths:
+        - contrib/pg_tde/src/smgr/**
+    - component_id: transam
+      paths:
+        - contrib/pg_tde/src/transam/**

.github/workflows/psp-reusable.yml → .github/workflows/build-and-test.yml

@@ -1,10 +1,13 @@
-name: PSP-Reusable
+name: Reusable build and test
 on:
   workflow_call:
     inputs:
       os:
         type: string
         required: true
+      compiler:
+        type: string
+        required: true
       build_type:
         type: string
         required: true
@@ -13,11 +16,12 @@ on:
         required: true
 
 env:
-  artifact_name: build-${{ inputs.os }}-${{ inputs.build_script }}-${{ inputs.build_type }}
+  artifact_name: build-${{ inputs.os }}-${{ inputs.compiler }}-${{ inputs.build_script }}-${{ inputs.build_type }}
+  CC: ${{ inputs.compiler }}
 
 jobs:
   build:
-    name: Build PSP
+    name: Build
     runs-on: ${{ inputs.os }}
     steps:
       - name: Clone repository
@@ -27,13 +31,19 @@ jobs:
           submodules: recursive
           ref: ${{ github.ref }}
 
+      # KMIP server don't support Python 3.12 for now: https://github.com/OpenKMIP/PyKMIP/pull/707 
+      - name: Downgrade python to 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
       - name: Install dependencies
         run: src/ci_scripts/ubuntu-deps.sh
 
       - name: Build postgres
         run: src/ci_scripts/${{ inputs.build_script }}-build.sh ${{ inputs.build_type }}
 
-      - name: 'Tar files'
+      - name: Archive pginst to artifact tar file
         run: tar -czf artifacts.tar src pginst
 
       - name: Upload build artifacts
@@ -46,7 +56,7 @@ jobs:
           retention-days: 1
 
   test:
-    name: Test PSP
+    name: Test
     runs-on: ${{ inputs.os }}
     needs: build
 
@@ -57,33 +67,45 @@ jobs:
           name: ${{ env.artifact_name }}
           path: .
 
-      - name: 'Untar files'
+      - name: Extract artifact file
         run: tar -xzf artifacts.tar
 
+      - name: Downgrade python to 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
       - name: Install dependencies
         run: src/ci_scripts/ubuntu-deps.sh
-  
+
       - name: Setup kmip and vault
         run: src/ci_scripts/setup-keyring-servers.sh
 
       - name: Test postgres
-        run: src/ci_scripts/${{ inputs.build_script }}-test.sh
+        run: src/ci_scripts/${{ inputs.build_script }}-test.sh all
 
       - name: Report on test fail
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: testlog-ubuntu-${{ inputs.ubuntu_version }}.04-meson-${{ inputs.build_type }}
+          name: log-test-${{ inputs.os }}-${{ inputs.compiler }}-${{ inputs.build_script }}-${{ inputs.build_type }}
           path: |
-            src/build/testrun/
-            src/contrib/*/t/
-            src/contrib/*/results
+            src/build/testrun
+            src/contrib/*/log
             src/contrib/*/regression.diffs
             src/contrib/*/regression.out
+            src/contrib/*/results
+            src/contrib/*/tmp_check
+            src/contrib/*/t/results
+            src/src/test/*/log
+            src/src/test/*/regression.diffs
+            src/src/test/*/regression.out
+            src/src/test/*/results
+            src/src/test/*/tmp_check
           retention-days: 3
 
-  test_tde:
-    name: Test PSP with TDE
+  test-default-tde:
+    name: Test with TDE as default access method
     runs-on: ${{ inputs.os }}
     if: inputs.build_script == 'make'
     needs: build
@@ -95,27 +117,40 @@ jobs:
           name: ${{ env.artifact_name }}
           path: .
 
-      - name: 'Untar files'
+      - name: Extract artifact file
         run: tar -xzf artifacts.tar
 
+      # KMIP server don't support Python 3.12 for now: https://github.com/OpenKMIP/PyKMIP/pull/707
+      - name: Downgrade python to 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
       - name: Install dependencies
-        run: src/ci_scripts/ubuntu-deps.sh 
-  
+        run: src/ci_scripts/ubuntu-deps.sh
+
       - name: Setup kmip and vault
         run: src/ci_scripts/setup-keyring-servers.sh
-  
-      - name: Test postgres with TDE
-        run: src/ci_scripts/${{ inputs.build_script }}-test-tde.sh --continue
+
+      - name: Test postgres with TDE as default access method
+        run: src/ci_scripts/${{ inputs.build_script }}-test-global-tde.sh --continue
 
       - name: Report on test fail
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: testlog-tde-${{ inputs.os }}-${{  inputs.build_script }}-${{ inputs.build_type }}
+          name: log-test-global-tde-${{ inputs.os }}-${{ inputs.compiler }}-${{ inputs.build_script }}-${{ inputs.build_type }}
           path: |
-            src/build/testrun/
-            src/contrib/*/t/
-            src/contrib/*/results
+            src/build/testrun
+            src/contrib/*/log
             src/contrib/*/regression.diffs
             src/contrib/*/regression.out
+            src/contrib/*/results
+            src/contrib/*/tmp_check
+            src/contrib/*/t/results
+            src/src/test/*/log
+            src/src/test/*/regression.diffs
+            src/src/test/*/regression.out
+            src/src/test/*/results
+            src/src/test/*/tmp_check
           retention-days: 3

.github/workflows/codechecker.yml

@@ -0,0 +1,57 @@
+name: CodeChecker
+on:
+  push:
+    branches:
+      - TDE_REL_17_STABLE
+    paths-ignore:
+      - contrib/pg_tde/documentation/**
+
+env:
+  CC: clang
+  LD: clang
+
+jobs:
+  run:
+    name: Run
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: ci_scripts/ubuntu-deps.sh
+
+      - name: Install CodeChecker
+        run: |
+          ## CodeChecker version should match version installed on server side.
+          pip3 install codechecker==6.26.1
+
+      - name: Configure CodeChecker
+        run: |
+          echo "::add-mask::${{ secrets.CODECHECKER_ENDPOINT }}"
+          echo "::add-mask::${{ secrets.CODECHECKER_ENGINEERING_CREDENTIALS }}"
+          cat > ~/.codechecker.passwords.json << EOL
+          {
+            "client_autologin": true,
+            "credentials": {
+              "${{secrets.CODECHECKER_ENDPOINT}}": "${{secrets.CODECHECKER_ENGINEERING_CREDENTIALS}}"
+            }
+          }
+          EOL
+
+      - name: Set cc alternative
+        run: |
+          sudo update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100
+          sudo update-alternatives --set cc /usr/bin/clang
+
+      - name: Build postgres
+        run: ci_scripts/meson-build.sh debug
+
+      - name: Run CodeChecker
+        run: CodeChecker analyze build/compile_commands.json --enable sensitive --output ./reports --file ${{ github.workspace }}/contrib/pg_tde
+
+      - name: Upload CodeChecker reports
+        run: |
+          CodeChecker store ./reports --url=https://codechecker.percona.com/pg_tde --name=${GITHUB_REF_NAME} --tag=${GITHUB_SHA} --force

.github/workflows/coverage.yml

@@ -0,0 +1,75 @@
+name: Code coverage
+on:
+  pull_request:
+    paths-ignore:
+      - contrib/pg_tde/documentation/**
+  push:
+    branches:
+      - TDE_REL_17_STABLE
+    paths-ignore:
+      - contrib/pg_tde/documentation/**
+
+env:
+  # Avoid failures on slow recovery
+  PGCTLTIMEOUT: 120
+  PG_TEST_TIMEOUT_DEFAULT: 300
+
+jobs:
+  collect:
+    name: Collect and upload
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      # KMIP server doesn't support Python 3.12 for now: https://github.com/OpenKMIP/PyKMIP/pull/707
+      - name: Downgrade python to 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: ci_scripts/ubuntu-deps.sh
+
+      - name: Build postgres
+        run: ci_scripts/make-build.sh debug --enable-coverage
+
+      - name: Setup kmip and vault
+        run: ci_scripts/setup-keyring-servers.sh
+
+      - name: Test postgres with TDE to generate coverage
+        run: ci_scripts/make-test.sh tde
+
+      - name: Collect coverage data
+        run: find . -type f -name "*.c" ! -path '*libkmip*' | xargs -t gcov -abcfu
+        working-directory: contrib/pg_tde
+
+      - name: Upload coverage data to codecov.io
+        uses: codecov/codecov-action@v5
+        with:
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+          working-directory: contrib/pg_tde
+          files: "*.c.gcov"
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: coverage-testlog-tde
+          path: |
+            build/testrun
+            contrib/*/log
+            contrib/*/regression.diffs
+            contrib/*/regression.out
+            contrib/*/results
+            contrib/*/tmp_check
+            contrib/*/t/results
+            src/test/*/log
+            src/test/*/regression.diffs
+            src/test/*/regression.out
+            src/test/*/results
+            src/test/*/tmp_check
+          retention-days: 3