fix: Remove mobile builds out of the template (#8000)

* fix: Remove mobile builds out of the template * fix: Use proper variables * fix: Create proper summary from detox output * fix: Test with free runners for now * fix: Switch to M1 * fix: Add removed node modules * fix: statsPercent * fix: Remove skipped from successful passes
2024-06-12 13:02:36 +03:00 · 2024-06-12 13:02:36 +03:00 · c1829bd24f
commit c1829bd24f
parent 5f01f9e9af
6 changed files with 147 additions and 54 deletions
--- a/.github/workflows/compatibility-matrix-testing.yml
+++ b/.github/workflows/compatibility-matrix-testing.yml
@ -68,11 +68,39 @@ jobs:
  #     }
  #   ]
  # }
+  build-ios-simulator:
+    runs-on: macos-14
+    needs:
+      - update-initial-status
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ inputs.MOBILE_VERSION }}
+
+      - name: Prepare iOS Build
+        uses: ./.github/actions/prepare-ios-build
+
+      - name: Build iOS Simulator
+        env:
+          TAG: "${{ inputs.MOBILE_VERSION }}"
+          AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}"
+          GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}"
+        run: bundle exec fastlane ios simulator --env ios.simulator
+        working-directory: ./fastlane
+
+      - name: Upload iOS Simulator Build
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: ios-build-simulator-${{ github.run_id }}
+          path: Mattermost-simulator-x86_64.app.zip
+
  detox-e2e:
    name: mobile-cmt-${{ matrix.server.version }}
    uses: ./.github/workflows/e2e-detox-template.yml
    needs:
-      - update-initial-status
+      - build-ios-simulator
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(inputs.CMT_MATRIX) }}
--- a/.github/workflows/e2e-detox-pr.yml
+++ b/.github/workflows/e2e-detox-pr.yml
@ -22,11 +22,37 @@ jobs:
          description: Detox tests for mattermost mobile app have started ...
          status: pending

+  build-ios-simulator:
+    runs-on: macos-14
+    needs:
+      - update-initial-status
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Prepare iOS Build
+        uses: ./.github/actions/prepare-ios-build
+
+      - name: Build iOS Simulator
+        env:
+          TAG: "${{ github.event.pull_request.head.sha }}"
+          AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}"
+          GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}"
+        run: bundle exec fastlane ios simulator --env ios.simulator
+        working-directory: ./fastlane
+
+      - name: Upload iOS Simulator Build
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: ios-build-simulator-${{ github.run_id }}
+          path: Mattermost-simulator-x86_64.app.zip
+
  run-ios-tests-on-pr:
    name: iOS Mobile Tests on PR
    uses: ./.github/workflows/e2e-detox-template.yml
    needs:
-      - update-initial-status
+      - build-ios-simulator
    with:
      run-ios-tests: true
      run-android-tests: false
--- a/.github/workflows/e2e-detox-release.yml
+++ b/.github/workflows/e2e-detox-release.yml
@ -19,10 +19,37 @@ jobs:
          description: Detox tests for mattermost mobile app have started ...
          status: pending

+  build-ios-simulator:
+    runs-on: macos-14
+    needs:
+      - update-initial-status
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Prepare iOS Build
+        uses: ./.github/actions/prepare-ios-build
+
+      - name: Build iOS Simulator
+        env:
+          TAG: "${{ github.ref }}"
+          AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}"
+          GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}"
+        run: bundle exec fastlane ios simulator --env ios.simulator
+        working-directory: ./fastlane
+
+      - name: Upload iOS Simulator Build
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: ios-build-simulator-${{ github.run_id }}
+          path: Mattermost-simulator-x86_64.app.zip
+
  run-ios-tests-on-release:
    name: iOS Mobile Tests on Release
    uses: ./.github/workflows/e2e-detox-template.yml
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release-') }}
+    needs:
+      - build-ios-simulator
    with:
      run-ios-tests: true
      run-android-tests: false
--- a/.github/workflows/e2e-detox-scheduled.yml
+++ b/.github/workflows/e2e-detox-scheduled.yml
@ -18,9 +18,37 @@ jobs:
          description: Detox tests for mattermost mobile app have started ...
          status: pending

+  build-ios-simulator:
+    runs-on: macos-14
+    needs:
+      - update-initial-status
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Prepare iOS Build
+        uses: ./.github/actions/prepare-ios-build
+
+      - name: Build iOS Simulator
+        env:
+          TAG: "${{ github.ref }}"
+          AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}"
+          GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}"
+        run: bundle exec fastlane ios simulator --env ios.simulator
+        working-directory: ./fastlane
+
+      - name: Upload iOS Simulator Build
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: ios-build-simulator-${{ github.run_id }}
+          path: Mattermost-simulator-x86_64.app.zip
+
  run-ios-tests-on-main-scheduled:
    name: iOS Mobile Tests on Main (Scheduled)
    uses: ./.github/workflows/e2e-detox-template.yml
+    needs:
+      - build-ios-simulator
    with:
      run-ios-tests: true
      run-android-tests: false
--- a/.github/workflows/e2e-detox-template.yml
+++ b/.github/workflows/e2e-detox-template.yml
@ -39,11 +39,11 @@ on:
        default: false
    outputs:
      STATUS:
-        value: ${{ jobs.download-e2e-results.outputs.STATUS }}
+        value: ${{ jobs.generate-report.outputs.STATUS }}
      TARGET_URL:
-        value: ${{ jobs.download-e2e-results.outputs.TARGET_URL }}
+        value: ${{ jobs.generate-report.outputs.TARGET_URL }}
      FAILURES:
-        value: ${{ jobs.download-e2e-results.outputs.FAILURES }}
+        value: ${{ jobs.generate-report.outputs.FAILURES }}

 env:
  AWS_REGION: "us-east-1"
@ -76,6 +76,7 @@ jobs:
      device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }}
      mobile_sha: ${{ steps.resolve-device.outputs.MOBILE_SHA }}
      mobile_ref: ${{ steps.resolve-device.outputs.MOBILE_REF }}
+      workflow_hash: ${{ steps.resolve-device.outputs.WORKFLOW_HASH }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -88,7 +89,11 @@ jobs:
          cd detox
          DEVICE_NAME=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.type")
          DEVICE_OS_VERSION=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.os")
-          BUILD_ID="${{ github.run_id }}-${DEVICE_NAME}-${DEVICE_OS_VERSION}"
+          WORKFLOW_HASH=$(tr -dc a-z0-9 </dev/urandom | head -c 10)
+          BUILD_ID="${{ github.run_id }}-${DEVICE_NAME}-${DEVICE_OS_VERSION}-${WORKFLOW_HASH}"
+
+          ## We need that hash to separate the artifacts
+          echo "WORKFLOW_HASH=${WORKFLOW_HASH}" >> ${GITHUB_OUTPUT}

          echo "BUILD_ID=$(echo ${BUILD_ID} | sed 's/ /_/g')" >> ${GITHUB_OUTPUT}
          echo "DEVICE_NAME=${DEVICE_NAME}" >> ${GITHUB_OUTPUT}
@ -105,46 +110,16 @@ jobs:
          device_name: ${{ steps.resolve-device.outputs.DEVICE_NAME }}
          device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }}

-  build-ios-simulator:
-    if: ${{ inputs.run-ios-tests }}
-    runs-on: macos-14-large
-    needs:
-      - generate-specs
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-        with:
-          ref: ${{ inputs.MOBILE_VERSION }}
-
-      - name: Prepare iOS Build
-        uses: ./.github/actions/prepare-ios-build
-
-      - name: Build iOS Simulator
-        env:
-          TAG: "${{ needs.generate-specs.outputs.mobile_ref }}"
-          AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}"
-          GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}"
-        run: bundle exec fastlane ios simulator --env ios.simulator
-        working-directory: ./fastlane
-
-      - name: Upload iOS Simulator Build
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
-        with:
-          name: ios-build-simulator-${{ github.run_id }}
-          path: Mattermost-simulator-x86_64.app.zip
-
  e2e-ios:
    name: ios-detox-e2e-${{ matrix.runId }}-${{ matrix.deviceName }}-${{ matrix.deviceOsVersion }}
    if: ${{ inputs.run-ios-tests }}
-    runs-on: macos-14-large
+    runs-on: macos-14
    continue-on-error: true
    timeout-minutes: 40
    env:
      IOS: true
    needs:
      - generate-specs
-      - build-ios-simulator
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-specs.outputs.specs) }}
@ -185,10 +160,10 @@ jobs:
        if: always()
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
-          name: ios-results-${{ matrix.runId }}
+          name: ios-results-${{ needs.generate-specs.outputs.workflow_hash }}-${{ matrix.runId }}
          path: detox/artifacts/

-  download-e2e-results:
+  generate-report:
    runs-on: ubuntu-22.04
    if: ${{ inputs.run-ios-tests || inputs.run-android-tests }}
    needs:
@ -197,7 +172,7 @@ jobs:
    outputs:
      TARGET_URL: ${{ steps.set-url.outputs.TARGET_URL }}
      STATUS: ${{ steps.determine-status.outputs.STATUS }}
-      FAILURES: ${{ steps.calculate-failures.outputs.FAILURES }}
+      FAILURES: ${{ steps.summary.outputs.FAILURES }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -207,14 +182,11 @@ jobs:
      - name: ci/prepare-node-deps
        uses: ./.github/actions/prepare-node-deps

-      - name: Create artifacts directory
-        run: mkdir -p detox/artifacts/
-
      - name: Download All Artifacts
        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
        with:
          path: detox/artifacts/
-          pattern: ios-results-*
+          pattern: ios-results-${{ needs.generate-specs.outputs.workflow_hash }}-*

      - name: Generate Report Path
        id: s3
@ -241,11 +213,14 @@ jobs:
          BRANCH: ${{ needs.generate-specs.outputs.mobile_ref }}

      - name: Calculate failures
-        id: calculate-failures
+        id: summary
        run: |
-          FAILURES=$(find detox/artifacts/ -name 'summary.json' | xargs -l jq -r '.stats.failures' | jq -s add)
-          echo "FAILURES=${FAILURES}" >> ${GITHUB_OUTPUT}
-          echo "Detox run completed with $FAILURES failures"
+          echo "FAILURES=$(cat detox/artifacts/summary.json | jq .stats.failures)" >> ${GITHUB_OUTPUT}
+          echo "PASSES=$(cat detox/artifacts/summary.json | jq .stats.passes)" >> ${GITHUB_OUTPUT}
+          echo "SKIPPED=$(cat detox/artifacts/summary.json | jq .stats.skipped)" >> ${GITHUB_OUTPUT}
+          echo "TOTAL=$(cat detox/artifacts/summary.json | jq .stats.tests)" >> ${GITHUB_OUTPUT}
+          echo "ERRORS=$(cat detox/artifacts/summary.json | jq .stats.errors)" >> ${GITHUB_OUTPUT}
+          echo "PERCENTAGE=$(cat detox/artifacts/summary.json | jq .stats.passPercent)" >> ${GITHUB_OUTPUT}

      - name: Set Target URL
        id: set-url
@ -259,8 +234,17 @@ jobs:
      - name: Determine Status
        id: determine-status
        run: |
-          if [[ ${{ steps.calculate-failures.outputs.failures }} -gt 0 && "${{ inputs.testcase_failure_fatal }}" == "true" ]]; then
+          if [[ ${{ steps.summary.outputs.failures }} -gt 0 && "${{ inputs.testcase_failure_fatal }}" == "true" ]]; then
            echo "STATUS=failure" >> ${GITHUB_OUTPUT}
          else
            echo "STATUS=success" >> ${GITHUB_OUTPUT}
          fi
+
+      - name: Generate Summary
+        run: |
+          echo "| Tests | Passed :white_check_mark: | Failed :x: | Skipped :fast_forward: | Errors :warning: | " >> ${GITHUB_STEP_SUMMARY}
+          echo "|:---:|:---:|:---:|:---:|:---:|" >> ${GITHUB_STEP_SUMMARY}
+          echo "| ${{ steps.summary.outputs.TOTAL }} | ${{ steps.summary.outputs.PASSES }} | ${{ steps.summary.outputs.FAILURES }} | ${{ steps.summary.outputs.SKIPPED }} | ${{ steps.summary.outputs.ERRORS }} |" >> ${GITHUB_STEP_SUMMARY}
+          echo "" >> ${GITHUB_STEP_SUMMARY}
+          echo "You can check the full report [here](${{ steps.set-url.outputs.TARGET_URL }})" >> ${GITHUB_STEP_SUMMARY}
+          echo "There was **${{ steps.summary.outputs.PERCENTAGE }}%** success rate." >> ${GITHUB_STEP_SUMMARY}
--- a/detox/utils/report.js
+++ b/detox/utils/report.js
@ -113,8 +113,8 @@ function generateStats(allTests) {
    const duration = allTests.duration;
    const start = allTests.start;
    const end = allTests.end;
-    const passes = tests - (failures + errors);
-    const passPercent = tests > 0 ? (passes / tests) * 100 : 0;
+    const passes = tests - (failures + errors + skipped);
+    const passPercent = tests > 0 ? ((passes / tests) * 100).toFixed(2) : 0;

    return {
        suites,
@ -134,7 +134,7 @@ function generateStatsFieldValue(stats, failedFullTitles) {
    let statsFieldValue = `
 | Key | Value |
 |:---|:---|
-| Passing Rate | ${stats.passPercent.toFixed(2)}% |
+| Passing Rate | ${stats.passPercent}% |
 | Duration | ${(stats.duration / (60 * 1000)).toFixed(4)} mins |
 | Suites | ${stats.suites} |
 | Tests | ${stats.tests} |
@ -281,7 +281,7 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te
        };
    }

-    let quickSummary = `${stats.passPercent.toFixed(2)}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`;
+    let quickSummary = `${stats.passPercent}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`;
    if (isUploadedToS3) {
        quickSummary = `[${quickSummary}](${reportLink})`;
    }