[benchmarks] Separate main and feature benchmarks.

petervdonovan · petervdonovan · commit 7bc8dfc37fa0 · 2022-07-13T23:39:53.000-07:00
diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml
@@ -0,0 +1,54 @@
+name: Run benchmarks
+description: Run benchmarks and collect results
+
+inputs:
+  scheduler:
+    type: string  # A comma-separated list (no spaces!) is allowed.
+    required: true
+  runtime-ref:
+    type: string
+    required: true
+
+
+outputs:
+  output-file:
+    description: "The CSV file summarizing all benchmark results"
+    value: ${{ steps.collect-results.outputs.output-file }}
+  
+
+runs:
+  using: "composite"
+  steps:
+    - name: Update LFC
+      run: |
+        echo $(ls)
+        cd lf/org.lflang/src/lib/c/reactor-c && \
+            git switch ${{ inputs.runtime-ref }} && cd ../../../../..
+        echo $(ls)
+        ./gradlew buildLfc
+        cd ..
+      shell: bash
+
+    - name: Run benchmarks
+      # TODO: Run all benchmarks with several numbers of threads
+      # run: |
+      #   ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \
+      #     benchmark="glob(*)" target=lf-c size=fast \
+      #     target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24
+      run: |  # DEBUG
+        ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \
+          benchmark="savina_concurrency_cigsmok" target=lf-c size=fast \
+          target.params.scheduler=${{ github.event.inputs.scheduler }} threads=1,3
+      shell: bash
+
+    - name: Collect benchmark results
+      id: collect-results
+      run: |
+        SHA=$(cd lf/org.lflang/src/lib/c/reactor-c && \
+          git rev-parse --short=7 HEAD && cd ../../../../..)
+        BRANCH=$(git rev-parse --abbrev-ref HEAD)
+        OUTPUT_FILE=${{ inputs.scheduler }}-$BRANCH-$SHA.csv
+        echo "The collected benchmark results will be saved to $OUTPUT_FILE"
+        ./runner/collect_results.py latest $OUTPUT_FILE
+        echo "::set-output name=output-file::$OUTPUT_FILE"
+      shell: bash
diff --git a/.github/actions/set-up-workspace/action.yml b/.github/actions/set-up-workspace/action.yml
@@ -27,7 +27,7 @@ runs:
       run: pip3 install -r runner/requirements.txt
       shell: bash
 
-    - name: Build lfc
+    - name: Build lfc  # FIXME: Redundant with run-benchmarks
       run: |
         cd lf
         ./gradlew buildLfc
@@ -37,4 +37,4 @@ runs:
       run: |
         echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV
         echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
-      shell: bash
+      shell: bash
diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml
@@ -7,8 +7,14 @@ on:
   workflow_dispatch:
     inputs:
       pr_number:
-        required: true
+        default: 91  # TODO: remove this line and uncomment required: true
+        # required: true
         type: number
+      scheduler:
+        default: NP
+      feature_branch:
+        default: automated_full_benchmark  # TODO (same as above)
+        # required: true
 
 permissions:
   contents: write
@@ -33,36 +39,45 @@ jobs:
           repository: lf-lang/reactor-c
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0  # It will be necessary to push to this repo
-          ref: automated-full-benchmark # FIXME: delete this line after merge
+          ref: graphics
           path: reactor-c
 
       - name: Set up workspace
         uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark
 
-      - name: Run C Benchmarks
-        # run: |
-        #   ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \
-        #     benchmark="glob(*)" target=lf-c size=fast \
-        #     target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24
-        run: |  # DEBUG
-          ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \
-            benchmark="savina_parallelism_recmatmul" target=lf-c size=fast \
-            target.params.scheduler=NP,adaptive threads=1,3,0
-        shell: bash
+      - name: Checkout benchmark repository
+        uses: actions/checkout@v2
+        with:
+          repository: lf-lang/benchmarks-lingua-franca
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 1
+          ref: main
 
-      - name: Collect benchmark results
-        run: |
-          ./runner/collect_results.py latest latest-benchmark-results.csv
-        shell: bash
+      - name: Run C Benchmarks (feature branch)
+        id: benchmarks-feature
+        uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark
+        with:
+          scheduler: ${{ github.event.inputs.scheduler }}
+          runtime-ref: ${{ github.event.inputs.branch }}
+
+      - name: Run C Benchmarks (main branch)
+        id: benchmarks-main
+        uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark
+        with:
+          scheduler: GEDF_NP,NP,adaptive
+          runtime-ref: main
 
       - name: Make figure
         run: |
           OUTPUT_DIR=images/$(date -I)
+          echo "Figure will be created in $OUTPUT_DIR"
           SHA=${{ github.sha }}
           OUTPUT_FILE=$OUTPUT_DIR/${SHA:0:7}.png
+          echo "Figure will be named $OUTPUT_FILE"
           mkdir -p reactor-c/$OUTPUT_DIR
-          ./runner/make-graphics.py latest-benchmark-results.csv reactor-c/$OUTPUT_FILE
-          echo "::set-output name=output_file::$OUTPUT_FILE"
+          ./runner/make-graphics.py ${{ steps.benchmarks-feature.outputs.output-file }} \
+            ${{ steps.benchmarks-main.outputs.output-file }} reactor-c/$OUTPUT_FILE
+          echo "::set-output name=output-file::$OUTPUT_FILE"
         shell: bash
         id: figure
 
@@ -72,8 +87,7 @@ jobs:
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --global user.name "github-actions[bot]"
           git fetch
-          git switch graphics
-          git add ${{ steps.figure.outputs.OUTPUT_FILE }}
+          git add ${{ steps.figure.outputs.output-file }}
           git commit -m "benchmark ${{ github.sha }}"
           git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \
             xargs -L1 git config --unset-all
@@ -86,5 +100,5 @@ jobs:
         with:
           message: |
             ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.OUTPUT_FILE }})
-          pr_number: 91 # ${{ inputs.pr_number }}
+          pr_number: ${{ inputs.pr_number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
