From 351c877a820161fe3bf0c041a8af9479ccdbd96a Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 26 Jun 2024 19:43:30 +0530 Subject: [PATCH] fix reporting variables in the community report (#7997) --- .../compatibility-matrix-testing.yml | 1 - .github/workflows/e2e-detox-pr.yml | 9 --- .github/workflows/e2e-detox-release.yml | 2 +- .github/workflows/e2e-detox-scheduled.yml | 4 +- .github/workflows/e2e-detox-template.yml | 66 +++++++++++-------- detox/.detoxrc.json | 4 +- detox/e2e/support/server_api/common.ts | 2 +- detox/package.json | 1 + detox/save_report.js | 15 ++--- detox/utils/artifacts.js | 7 +- detox/utils/generate_detox_config_ci.js | 14 ++++ detox/utils/report.js | 4 +- 12 files changed, 71 insertions(+), 58 deletions(-) create mode 100644 detox/utils/generate_detox_config_ci.js diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 84b878f9c..7b7cac9e2 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -107,7 +107,6 @@ jobs: secrets: inherit with: run-ios-tests: true - run-android-tests: false run-type: "RELEASE" MM_TEST_SERVER_URL: ${{ matrix.server.url }} MOBILE_VERSION: ${{ inputs.MOBILE_VERSION }} diff --git a/.github/workflows/e2e-detox-pr.yml b/.github/workflows/e2e-detox-pr.yml index 571ee8b14..a36299dad 100644 --- a/.github/workflows/e2e-detox-pr.yml +++ b/.github/workflows/e2e-detox-pr.yml @@ -55,7 +55,6 @@ jobs: - build-ios-simulator with: run-ios-tests: true - run-android-tests: false run-type: "PR" MOBILE_VERSION: ${{ github.event.pull_request.head.sha }} secrets: inherit @@ -99,11 +98,3 @@ jobs: }); } - if (labels.includes(androidLabel)) { - github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: androidLabel, - }); - } diff --git a/.github/workflows/e2e-detox-release.yml b/.github/workflows/e2e-detox-release.yml index 140d29998..bb604de60 100644 --- a/.github/workflows/e2e-detox-release.yml +++ b/.github/workflows/e2e-detox-release.yml @@ -52,8 +52,8 @@ jobs: - build-ios-simulator with: run-ios-tests: true - run-android-tests: false run-type: "RELEASE" + record_tests_in_zephyr: 'true' MOBILE_VERSION: ${{ github.ref }} secrets: inherit diff --git a/.github/workflows/e2e-detox-scheduled.yml b/.github/workflows/e2e-detox-scheduled.yml index c9f28aa42..10786f4d6 100644 --- a/.github/workflows/e2e-detox-scheduled.yml +++ b/.github/workflows/e2e-detox-scheduled.yml @@ -2,7 +2,7 @@ name: Detox E2E Tests (Scheduled) on: schedule: - - cron: "0 0 * * *" # every day at midnight + - cron: "0 0 * * 4,5" # Wednesday and Thursday midnight jobs: update-initial-status: @@ -51,8 +51,8 @@ jobs: - build-ios-simulator with: run-ios-tests: true - run-android-tests: false run-type: "MAIN" + record_tests_in_zephyr: 'true' MOBILE_VERSION: ${{ github.ref }} secrets: inherit diff --git a/.github/workflows/e2e-detox-template.yml b/.github/workflows/e2e-detox-template.yml index 70718e5f7..bf3c4102f 100644 --- a/.github/workflows/e2e-detox-template.yml +++ b/.github/workflows/e2e-detox-template.yml @@ -24,10 +24,6 @@ on: description: "Run iOS tests" required: true type: boolean - run-android-tests: - description: "Run Android tests" - required: true - type: boolean run-type: type: string required: false @@ -37,6 +33,21 @@ on: required: false type: boolean default: false + record_tests_in_zephyr: + description: "Record test results in Zephyr, typically for nightly and release runs" + required: false + type: string + default: 'false' + ios_device_name: + description: "iPhone simulator name" + required: false + type: string + default: "iPhone 14" + ios_device_os_name: + description: "iPhone simulator OS version" + required: false + type: string + default: "iOS 17.0" outputs: STATUS: value: ${{ jobs.generate-report.outputs.STATUS }} @@ -48,16 +59,20 @@ on: env: AWS_REGION: "us-east-1" ADMIN_EMAIL: ${{ secrets.MM_MOBILE_E2E_ADMIN_EMAIL }} - ADMIN_USERNAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_MOBILE_E2E_ADMIN_USERNAME }} - ADMIN_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_MOBILE_E2E_ADMIN_PASSWORD }} + ADMIN_USERNAME: ${{ secrets.MM_MOBILE_E2E_ADMIN_USERNAME }} + ADMIN_PASSWORD: ${{ secrets.MM_MOBILE_E2E_ADMIN_PASSWORD }} + # BRANCH: ${{ github.head_ref || github.ref_name }} + # COMMIT_HASH: ${{ github.sha }} + DEVICE_NAME: ${{ inputs.ios_device_name }} + DEVICE_OS_VERSION: ${{ inputs.ios_device_os_name }} DETOX_AWS_S3_BUCKET: "mattermost-detox-report" HEADLESS: "true" TYPE: ${{ inputs.run-type }} PULL_REQUEST: "https://github.com/mattermost/mattermost-mobile/pull/${{ github.event.number }}" - SITE_1_URL: ${{ inputs.MM_TEST_SERVER_URL || secrets.MM_MOBILE_E2E_SITE_1_URL }} - SITE_2_URL: ${{ secrets.MM_MOBILE_E2E_SITE_2_URL }} - SITE_3_URL: ${{ secrets.MM_MOBILE_E2E_SITE_3_URL }} - ZEPHYR_ENABLE: false + SITE_1_URL: ${{ inputs.MM_TEST_SERVER_URL || 'https://mobile-e2e-site-1.test.mattermost.cloud' }} + SITE_2_URL: "https://mobile-e2e-site-2.test.mattermost.cloud" + SITE_3_URL: "https://mobile-e2e-site-3.test.mattermost.cloud" + ZEPHYR_ENABLE: ${{ inputs.record_tests_in_zephyr }} JIRA_PROJECT_KEY: "MM" ZEPHYR_API_KEY: ${{ secrets.MM_MOBILE_E2E_ZEPHYR_API_KEY }} ZEPHYR_FOLDER_ID: "3233873" @@ -68,12 +83,10 @@ env: jobs: generate-specs: runs-on: ubuntu-22.04 - if: ${{ inputs.run-ios-tests || inputs.run-android-tests }} + if: ${{ inputs.run-ios-tests }} outputs: specs: ${{ steps.generate-specs.outputs.specs }} build_id: ${{ steps.resolve-device.outputs.BUILD_ID }} - device_name: ${{ steps.resolve-device.outputs.DEVICE_NAME }} - device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }} mobile_sha: ${{ steps.resolve-device.outputs.MOBILE_SHA }} mobile_ref: ${{ steps.resolve-device.outputs.MOBILE_REF }} workflow_hash: ${{ steps.resolve-device.outputs.WORKFLOW_HASH }} @@ -86,18 +99,13 @@ jobs: - name: Set Build ID id: resolve-device run: | - cd detox - DEVICE_NAME=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.type") - DEVICE_OS_VERSION=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.os") + BUILD_ID="${{ github.run_id }}-${{ env.DEVICE_NAME }}-${{ env.DEVICE_OS_VERSION}}" WORKFLOW_HASH=$(tr -dc a-z0-9 > ${GITHUB_OUTPUT} echo "BUILD_ID=$(echo ${BUILD_ID} | sed 's/ /_/g')" >> ${GITHUB_OUTPUT} - echo "DEVICE_NAME=${DEVICE_NAME}" >> ${GITHUB_OUTPUT} - echo "DEVICE_OS_VERSION=${DEVICE_OS_VERSION}" >> ${GITHUB_OUTPUT} echo "MOBILE_SHA=$(git rev-parse HEAD)" >> ${GITHUB_OUTPUT} echo "MOBILE_REF=$(git rev-parse --abbrev-ref HEAD)" >> ${GITHUB_OUTPUT} @@ -107,8 +115,8 @@ jobs: with: parallelism: 10 search_path: detox/e2e/test - device_name: ${{ steps.resolve-device.outputs.DEVICE_NAME }} - device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }} + device_name: ${{ env.DEVICE_NAME }} + device_os_version: ${{ env.DEVICE_OS_VERSION }} e2e-ios: name: ios-detox-e2e-${{ matrix.runId }}-${{ matrix.deviceName }}-${{ matrix.deviceOsVersion }} @@ -154,7 +162,13 @@ jobs: - name: Run Detox E2E Tests continue-on-error: true # Label might have been removed manually - run: cd detox && npm run e2e:ios-test -- ${{ matrix.specs }} + run: | + cd detox + npm run detox:config-gen + npm run e2e:ios-test -- ${{ matrix.specs }} + env: + DETOX_DEVICE_TYPE: ${{ env.DEVICE_NAME }} + DETOX_OS_VERSION: ${{ env.DEVICE_OS_VERSION }} - name: Upload iOS Test Report if: always() @@ -165,7 +179,7 @@ jobs: generate-report: runs-on: ubuntu-22.04 - if: ${{ inputs.run-ios-tests || inputs.run-android-tests }} + if: ${{ inputs.run-ios-tests}} needs: - generate-specs - e2e-ios @@ -205,8 +219,6 @@ jobs: DETOX_AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_MOBILE_DETOX_AWS_SECRET_ACCESS_KEY }} IOS: ${{ inputs.run-ios-tests }} BUILD_ID: ${{ needs.generate-specs.outputs.build_id }} - DEVICE_NAME: ${{ needs.generate-specs.outputs.device_name }} - DEVICE_OS_VERSION: ${{ needs.generate-specs.outputs.device_os_version }} REPORT_PATH: ${{ steps.s3.outputs.path }} ## These are needed for the MM Webhook report COMMIT_HASH: ${{ needs.generate-specs.outputs.mobile_sha }} @@ -225,11 +237,7 @@ jobs: - name: Set Target URL id: set-url run: | - if ${{ inputs.run-ios-tests }}; then echo "TARGET_URL=https://${{ env.DETOX_AWS_S3_BUCKET }}.s3.amazonaws.com/${{ steps.s3.outputs.path }}/jest-stare/ios-report.html" >> ${GITHUB_OUTPUT} - else - echo "TARGET_URL=https://${{ env.DETOX_AWS_S3_BUCKET }}.s3.amazonaws.com/${{ steps.s3.outputs.path }}/jest-stare/android-report.html" >> ${GITHUB_OUTPUT} - fi - name: Determine Status id: determine-status diff --git a/detox/.detoxrc.json b/detox/.detoxrc.json index c6a08e53f..d57c9abda 100644 --- a/detox/.detoxrc.json +++ b/detox/.detoxrc.json @@ -30,8 +30,8 @@ "ios.simulator": { "type": "ios.simulator", "device": { - "type": "iPhone 15 Pro", - "os": "iOS 17.4" + "type": "__DEVICE_TYPE__", + "os": "__DEVICE_OS__" } }, "android.emulator": { diff --git a/detox/e2e/support/server_api/common.ts b/detox/e2e/support/server_api/common.ts index d4060eec1..5cfef1fd9 100644 --- a/detox/e2e/support/server_api/common.ts +++ b/detox/e2e/support/server_api/common.ts @@ -32,7 +32,7 @@ export const getCookiesFromConfig = (config: AxiosRequestConfig) => { export const getResponseFromError = (err: any) => { const {response} = err; if (!response) { - const message = `No response from server at "${err.config.baseURL}". + const message = `No response from server at "${err}". If testing against a server other than the default local instance, you may set the server URL via "SITE_URL" environment variable. `; diff --git a/detox/package.json b/detox/package.json index 4190e2d1c..9e4536b07 100644 --- a/detox/package.json +++ b/detox/package.json @@ -51,6 +51,7 @@ "e2e:ios-test": "IOS=true detox test -c ios.sim.debug", "e2e:ios-build-release": "detox build -c ios.sim.release", "e2e:ios-test-release": "IOS=true detox test -c ios.sim.release --record-logs failing --take-screenshots failing", + "detox:config-gen": "cd utils && node generate_detox_config_ci.js", "check": "npm run lint && npm run tsc", "clean-detox": "detox clean", "lint": "eslint --ignore-pattern node_modules --quiet .", diff --git a/detox/save_report.js b/detox/save_report.js index 2de5c6b5b..8845d5b22 100644 --- a/detox/save_report.js +++ b/detox/save_report.js @@ -69,7 +69,6 @@ const saveReport = async () => { WEBHOOK_URL, ZEPHYR_ENABLE, ZEPHYR_CYCLE_KEY, - REPORT_PATH, } = process.env; // Remove old generated reports @@ -100,7 +99,7 @@ const saveReport = async () => { // Merge all XML reports into one single XML report const platform = process.env.IOS === 'true' ? 'ios' : 'android'; const combinedFilePath = `${ARTIFACTS_DIR}/${platform}-combined.xml`; - await mergeFiles(path.join(__dirname, combinedFilePath), [`${ARTIFACTS_DIR}/ios-results*/${platform}-junit*.xml`]); + await mergeFiles(path.join(__dirname, combinedFilePath), [`${ARTIFACTS_DIR}/${platform}-results*/${platform}-junit*.xml`]); console.log(`Merged, check ${combinedFilePath}`); // Read XML from a file @@ -120,27 +119,27 @@ const saveReport = async () => { fs.mkdirSync(jestStareOutputDir, {recursive: true}); } - // 'ios-results-*' is the path in CI where the parallel detox jobs save artifacts - await mergeJestStareJsonFiles(jestStareCombinedFilePath, [`${ARTIFACTS_DIR}/ios-results*/jest-stare/${platform}-data*.json`]); + // "ios-results-*" or "android-results-*" is the path in CI where the parallel detox jobs save the artifacts + await mergeJestStareJsonFiles(jestStareCombinedFilePath, [`${ARTIFACTS_DIR}/${platform}-results*/jest-stare/${platform}-data*.json`]); generateJestStareHtmlReport(jestStareOutputDir, `${platform}-report.html`, jestStareCombinedFilePath); if (process.env.CI) { - // Delete folders starting with "iOS-results-" only in CI environment + // Delete folders starting with "ios-results-" or "android-results-" only in CI environment const entries = fs.readdirSync(ARTIFACTS_DIR, {withFileTypes: true}); for (const entry of entries) { - if (entry.isDirectory() && entry.name.startsWith('ios-results-')) { + if (entry.isDirectory() && entry.name.startsWith(`${platform}-results-`)) { fs.rmSync(path.join(ARTIFACTS_DIR, entry.name), {recursive: true}); } } } - const result = await saveArtifacts(REPORT_PATH); + const result = await saveArtifacts(); if (result && result.success) { console.log('Successfully uploaded artifacts to S3:', result.reportLink); } // Create or use an existing test cycle let testCycle = {}; - if (ZEPHYR_ENABLE === 'true') { + if (ZEPHYR_ENABLE === true) { const {start, end} = summary.stats; testCycle = ZEPHYR_CYCLE_KEY ? {key: ZEPHYR_CYCLE_KEY} : await createTestCycle(start, end); } diff --git a/detox/utils/artifacts.js b/detox/utils/artifacts.js index 431d3c3da..0778defb0 100644 --- a/detox/utils/artifacts.js +++ b/detox/utils/artifacts.js @@ -21,6 +21,7 @@ const { DETOX_AWS_ACCESS_KEY_ID, DETOX_AWS_SECRET_ACCESS_KEY, IOS, + REPORT_PATH, } = process.env; const platform = IOS === 'true' ? 'ios' : 'android'; @@ -35,7 +36,7 @@ function getFiles(dirPath) { return fs.existsSync(dirPath) ? readdir(dirPath) : []; } -async function saveArtifacts(s3Folder) { +async function saveArtifacts() { if (!DETOX_AWS_S3_BUCKET || !DETOX_AWS_ACCESS_KEY_ID || !DETOX_AWS_SECRET_ACCESS_KEY) { console.log('No AWS credentials found. Test artifacts not uploaded to S3.'); @@ -50,7 +51,7 @@ async function saveArtifacts(s3Folder) { filesToUpload, 10, async.asyncify(async (file) => { - const Key = file.replace(uploadPath, s3Folder); + const Key = file.replace(uploadPath, REPORT_PATH); const contentType = mime.lookup(file); const charset = mime.charset(contentType); @@ -76,7 +77,7 @@ async function saveArtifacts(s3Folder) { return reject(new Error(err)); } - const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/jest-stare/${platform}-report.html`; + const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${REPORT_PATH}/jest-stare/${platform}-report.html`; resolve({success: true, reportLink}); }, ); diff --git a/detox/utils/generate_detox_config_ci.js b/detox/utils/generate_detox_config_ci.js new file mode 100644 index 000000000..abf734230 --- /dev/null +++ b/detox/utils/generate_detox_config_ci.js @@ -0,0 +1,14 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +/* eslint-disable no-process-env */ +/* eslint-disable no-console */ +const fs = require('fs'); +const deviceType = process.env.DETOX_DEVICE_TYPE || 'iPhone 15 Pro'; +const deviceOS = process.env.DETOX_DEVICE_OS || 'iOS 17.4'; +const detoxConfigTemplate = fs.readFileSync('../.detoxrc.json', 'utf8'); +const detoxConfig = detoxConfigTemplate.replace('__DEVICE_TYPE__', deviceType).replace('__DEVICE_OS__', deviceOS); + +fs.writeFileSync('../.detoxrc.json', detoxConfig); + +console.log('Detox configuration generated successfully'); diff --git a/detox/utils/report.js b/detox/utils/report.js index 649da2860..c0bc53fb9 100644 --- a/detox/utils/report.js +++ b/detox/utils/report.js @@ -308,8 +308,8 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te function generateTitle() { const { BRANCH, - DETOX_AWS_S3_BUCKET, COMMIT_HASH, + DETOX_AWS_S3_BUCKET, IOS, PULL_REQUEST, RELEASE_BUILD_NUMBER, @@ -324,7 +324,7 @@ function generateTitle() { const appExtension = IOS === 'true' ? 'ipa' : 'apk'; const appFileName = `Mattermost_Beta.${appExtension}`; const appBuildType = 'mattermost-mobile-beta'; - const s3Folder = `${platform.toLocaleLowerCase()}/${REPORT_PATH}`.replace(/\./g, '-'); + const s3Folder = `${platform.toLocaleLowerCase()}/${REPORT_PATH}`; const appFilePath = IOS === 'true' ? 'Mattermost-simulator-x86_64.app.zip' : 'android/app/build/outputs/apk/release/app-release.apk'; let buildLink = ''; let releaseDate = '';