From c1829bd24fa4500306f7385cde858d2d441bba5f Mon Sep 17 00:00:00 2001 From: Antonis Stamatiou Date: Wed, 12 Jun 2024 13:02:36 +0300 Subject: [PATCH] fix: Remove mobile builds out of the template (#8000) * fix: Remove mobile builds out of the template * fix: Use proper variables * fix: Create proper summary from detox output * fix: Test with free runners for now * fix: Switch to M1 * fix: Add removed node modules * fix: statsPercent * fix: Remove skipped from successful passes --- .../compatibility-matrix-testing.yml | 30 ++++++- .github/workflows/e2e-detox-pr.yml | 28 ++++++- .github/workflows/e2e-detox-release.yml | 29 ++++++- .github/workflows/e2e-detox-scheduled.yml | 28 +++++++ .github/workflows/e2e-detox-template.yml | 78 ++++++++----------- detox/utils/report.js | 8 +- 6 files changed, 147 insertions(+), 54 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 2b558e7ce..84b878f9c 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -68,11 +68,39 @@ jobs: # } # ] # } + build-ios-simulator: + runs-on: macos-14 + needs: + - update-initial-status + steps: + - name: Checkout Repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.MOBILE_VERSION }} + + - name: Prepare iOS Build + uses: ./.github/actions/prepare-ios-build + + - name: Build iOS Simulator + env: + TAG: "${{ inputs.MOBILE_VERSION }}" + AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}" + GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}" + run: bundle exec fastlane ios simulator --env ios.simulator + working-directory: ./fastlane + + - name: Upload iOS Simulator Build + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: ios-build-simulator-${{ github.run_id }} + path: Mattermost-simulator-x86_64.app.zip + detox-e2e: name: mobile-cmt-${{ matrix.server.version }} uses: ./.github/workflows/e2e-detox-template.yml needs: - - update-initial-status + - build-ios-simulator strategy: fail-fast: false matrix: ${{ fromJson(inputs.CMT_MATRIX) }} diff --git a/.github/workflows/e2e-detox-pr.yml b/.github/workflows/e2e-detox-pr.yml index aa9088750..571ee8b14 100644 --- a/.github/workflows/e2e-detox-pr.yml +++ b/.github/workflows/e2e-detox-pr.yml @@ -22,11 +22,37 @@ jobs: description: Detox tests for mattermost mobile app have started ... status: pending + build-ios-simulator: + runs-on: macos-14 + needs: + - update-initial-status + steps: + - name: Checkout Repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Prepare iOS Build + uses: ./.github/actions/prepare-ios-build + + - name: Build iOS Simulator + env: + TAG: "${{ github.event.pull_request.head.sha }}" + AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}" + GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}" + run: bundle exec fastlane ios simulator --env ios.simulator + working-directory: ./fastlane + + - name: Upload iOS Simulator Build + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: ios-build-simulator-${{ github.run_id }} + path: Mattermost-simulator-x86_64.app.zip + run-ios-tests-on-pr: name: iOS Mobile Tests on PR uses: ./.github/workflows/e2e-detox-template.yml needs: - - update-initial-status + - build-ios-simulator with: run-ios-tests: true run-android-tests: false diff --git a/.github/workflows/e2e-detox-release.yml b/.github/workflows/e2e-detox-release.yml index 17fb94aba..140d29998 100644 --- a/.github/workflows/e2e-detox-release.yml +++ b/.github/workflows/e2e-detox-release.yml @@ -19,10 +19,37 @@ jobs: description: Detox tests for mattermost mobile app have started ... status: pending + build-ios-simulator: + runs-on: macos-14 + needs: + - update-initial-status + steps: + - name: Checkout Repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Prepare iOS Build + uses: ./.github/actions/prepare-ios-build + + - name: Build iOS Simulator + env: + TAG: "${{ github.ref }}" + AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}" + GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}" + run: bundle exec fastlane ios simulator --env ios.simulator + working-directory: ./fastlane + + - name: Upload iOS Simulator Build + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: ios-build-simulator-${{ github.run_id }} + path: Mattermost-simulator-x86_64.app.zip + run-ios-tests-on-release: name: iOS Mobile Tests on Release uses: ./.github/workflows/e2e-detox-template.yml - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release-') }} + needs: + - build-ios-simulator with: run-ios-tests: true run-android-tests: false diff --git a/.github/workflows/e2e-detox-scheduled.yml b/.github/workflows/e2e-detox-scheduled.yml index 9734956c6..c9f28aa42 100644 --- a/.github/workflows/e2e-detox-scheduled.yml +++ b/.github/workflows/e2e-detox-scheduled.yml @@ -18,9 +18,37 @@ jobs: description: Detox tests for mattermost mobile app have started ... status: pending + build-ios-simulator: + runs-on: macos-14 + needs: + - update-initial-status + steps: + - name: Checkout Repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Prepare iOS Build + uses: ./.github/actions/prepare-ios-build + + - name: Build iOS Simulator + env: + TAG: "${{ github.ref }}" + AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}" + GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}" + run: bundle exec fastlane ios simulator --env ios.simulator + working-directory: ./fastlane + + - name: Upload iOS Simulator Build + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: ios-build-simulator-${{ github.run_id }} + path: Mattermost-simulator-x86_64.app.zip + run-ios-tests-on-main-scheduled: name: iOS Mobile Tests on Main (Scheduled) uses: ./.github/workflows/e2e-detox-template.yml + needs: + - build-ios-simulator with: run-ios-tests: true run-android-tests: false diff --git a/.github/workflows/e2e-detox-template.yml b/.github/workflows/e2e-detox-template.yml index 94b2f07d1..70718e5f7 100644 --- a/.github/workflows/e2e-detox-template.yml +++ b/.github/workflows/e2e-detox-template.yml @@ -39,11 +39,11 @@ on: default: false outputs: STATUS: - value: ${{ jobs.download-e2e-results.outputs.STATUS }} + value: ${{ jobs.generate-report.outputs.STATUS }} TARGET_URL: - value: ${{ jobs.download-e2e-results.outputs.TARGET_URL }} + value: ${{ jobs.generate-report.outputs.TARGET_URL }} FAILURES: - value: ${{ jobs.download-e2e-results.outputs.FAILURES }} + value: ${{ jobs.generate-report.outputs.FAILURES }} env: AWS_REGION: "us-east-1" @@ -76,6 +76,7 @@ jobs: device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }} mobile_sha: ${{ steps.resolve-device.outputs.MOBILE_SHA }} mobile_ref: ${{ steps.resolve-device.outputs.MOBILE_REF }} + workflow_hash: ${{ steps.resolve-device.outputs.WORKFLOW_HASH }} steps: - name: Checkout Repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -88,7 +89,11 @@ jobs: cd detox DEVICE_NAME=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.type") DEVICE_OS_VERSION=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.os") - BUILD_ID="${{ github.run_id }}-${DEVICE_NAME}-${DEVICE_OS_VERSION}" + WORKFLOW_HASH=$(tr -dc a-z0-9 > ${GITHUB_OUTPUT} echo "BUILD_ID=$(echo ${BUILD_ID} | sed 's/ /_/g')" >> ${GITHUB_OUTPUT} echo "DEVICE_NAME=${DEVICE_NAME}" >> ${GITHUB_OUTPUT} @@ -105,46 +110,16 @@ jobs: device_name: ${{ steps.resolve-device.outputs.DEVICE_NAME }} device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }} - build-ios-simulator: - if: ${{ inputs.run-ios-tests }} - runs-on: macos-14-large - needs: - - generate-specs - steps: - - name: Checkout Repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ inputs.MOBILE_VERSION }} - - - name: Prepare iOS Build - uses: ./.github/actions/prepare-ios-build - - - name: Build iOS Simulator - env: - TAG: "${{ needs.generate-specs.outputs.mobile_ref }}" - AWS_ACCESS_KEY_ID: "${{ secrets.MM_MOBILE_BETA_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.MM_MOBILE_BETA_AWS_SECRET_ACCESS_KEY }}" - GITHUB_TOKEN: "${{ secrets.MM_MOBILE_GITHUB_TOKEN }}" - run: bundle exec fastlane ios simulator --env ios.simulator - working-directory: ./fastlane - - - name: Upload iOS Simulator Build - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - with: - name: ios-build-simulator-${{ github.run_id }} - path: Mattermost-simulator-x86_64.app.zip - e2e-ios: name: ios-detox-e2e-${{ matrix.runId }}-${{ matrix.deviceName }}-${{ matrix.deviceOsVersion }} if: ${{ inputs.run-ios-tests }} - runs-on: macos-14-large + runs-on: macos-14 continue-on-error: true timeout-minutes: 40 env: IOS: true needs: - generate-specs - - build-ios-simulator strategy: fail-fast: false matrix: ${{ fromJSON(needs.generate-specs.outputs.specs) }} @@ -185,10 +160,10 @@ jobs: if: always() uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: ios-results-${{ matrix.runId }} + name: ios-results-${{ needs.generate-specs.outputs.workflow_hash }}-${{ matrix.runId }} path: detox/artifacts/ - download-e2e-results: + generate-report: runs-on: ubuntu-22.04 if: ${{ inputs.run-ios-tests || inputs.run-android-tests }} needs: @@ -197,7 +172,7 @@ jobs: outputs: TARGET_URL: ${{ steps.set-url.outputs.TARGET_URL }} STATUS: ${{ steps.determine-status.outputs.STATUS }} - FAILURES: ${{ steps.calculate-failures.outputs.FAILURES }} + FAILURES: ${{ steps.summary.outputs.FAILURES }} steps: - name: Checkout Repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -207,14 +182,11 @@ jobs: - name: ci/prepare-node-deps uses: ./.github/actions/prepare-node-deps - - name: Create artifacts directory - run: mkdir -p detox/artifacts/ - - name: Download All Artifacts uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 with: path: detox/artifacts/ - pattern: ios-results-* + pattern: ios-results-${{ needs.generate-specs.outputs.workflow_hash }}-* - name: Generate Report Path id: s3 @@ -241,11 +213,14 @@ jobs: BRANCH: ${{ needs.generate-specs.outputs.mobile_ref }} - name: Calculate failures - id: calculate-failures + id: summary run: | - FAILURES=$(find detox/artifacts/ -name 'summary.json' | xargs -l jq -r '.stats.failures' | jq -s add) - echo "FAILURES=${FAILURES}" >> ${GITHUB_OUTPUT} - echo "Detox run completed with $FAILURES failures" + echo "FAILURES=$(cat detox/artifacts/summary.json | jq .stats.failures)" >> ${GITHUB_OUTPUT} + echo "PASSES=$(cat detox/artifacts/summary.json | jq .stats.passes)" >> ${GITHUB_OUTPUT} + echo "SKIPPED=$(cat detox/artifacts/summary.json | jq .stats.skipped)" >> ${GITHUB_OUTPUT} + echo "TOTAL=$(cat detox/artifacts/summary.json | jq .stats.tests)" >> ${GITHUB_OUTPUT} + echo "ERRORS=$(cat detox/artifacts/summary.json | jq .stats.errors)" >> ${GITHUB_OUTPUT} + echo "PERCENTAGE=$(cat detox/artifacts/summary.json | jq .stats.passPercent)" >> ${GITHUB_OUTPUT} - name: Set Target URL id: set-url @@ -259,8 +234,17 @@ jobs: - name: Determine Status id: determine-status run: | - if [[ ${{ steps.calculate-failures.outputs.failures }} -gt 0 && "${{ inputs.testcase_failure_fatal }}" == "true" ]]; then + if [[ ${{ steps.summary.outputs.failures }} -gt 0 && "${{ inputs.testcase_failure_fatal }}" == "true" ]]; then echo "STATUS=failure" >> ${GITHUB_OUTPUT} else echo "STATUS=success" >> ${GITHUB_OUTPUT} fi + + - name: Generate Summary + run: | + echo "| Tests | Passed :white_check_mark: | Failed :x: | Skipped :fast_forward: | Errors :warning: | " >> ${GITHUB_STEP_SUMMARY} + echo "|:---:|:---:|:---:|:---:|:---:|" >> ${GITHUB_STEP_SUMMARY} + echo "| ${{ steps.summary.outputs.TOTAL }} | ${{ steps.summary.outputs.PASSES }} | ${{ steps.summary.outputs.FAILURES }} | ${{ steps.summary.outputs.SKIPPED }} | ${{ steps.summary.outputs.ERRORS }} |" >> ${GITHUB_STEP_SUMMARY} + echo "" >> ${GITHUB_STEP_SUMMARY} + echo "You can check the full report [here](${{ steps.set-url.outputs.TARGET_URL }})" >> ${GITHUB_STEP_SUMMARY} + echo "There was **${{ steps.summary.outputs.PERCENTAGE }}%** success rate." >> ${GITHUB_STEP_SUMMARY} diff --git a/detox/utils/report.js b/detox/utils/report.js index ab7fd95d1..649da2860 100644 --- a/detox/utils/report.js +++ b/detox/utils/report.js @@ -113,8 +113,8 @@ function generateStats(allTests) { const duration = allTests.duration; const start = allTests.start; const end = allTests.end; - const passes = tests - (failures + errors); - const passPercent = tests > 0 ? (passes / tests) * 100 : 0; + const passes = tests - (failures + errors + skipped); + const passPercent = tests > 0 ? ((passes / tests) * 100).toFixed(2) : 0; return { suites, @@ -134,7 +134,7 @@ function generateStatsFieldValue(stats, failedFullTitles) { let statsFieldValue = ` | Key | Value | |:---|:---| -| Passing Rate | ${stats.passPercent.toFixed(2)}% | +| Passing Rate | ${stats.passPercent}% | | Duration | ${(stats.duration / (60 * 1000)).toFixed(4)} mins | | Suites | ${stats.suites} | | Tests | ${stats.tests} | @@ -281,7 +281,7 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te }; } - let quickSummary = `${stats.passPercent.toFixed(2)}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`; + let quickSummary = `${stats.passPercent}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`; if (isUploadedToS3) { quickSummary = `[${quickSummary}](${reportLink})`; }