From abb526cf8335353845090fd8bbdf0c18c89d3ff9 Mon Sep 17 00:00:00 2001 From: Rahim Rahman Date: Fri, 14 Mar 2025 13:21:25 -0600 Subject: [PATCH] ci(MM-63199): code coverage tracking (#8642) * ci(MM-63199): code coverage tracking * try to download existing coverage file * read coverage * add token * use github.token instead * passing github token * github_token passing from workflow * remove download * re-add download * wrong param * try download all artifacts * add run-id so to retrieve with download later * remove read coverage temp * use run-id to download * put files into current-coverage * using last run id * temporary comment * can retrieve last run id? * remove hard-coding * echo into github_env vs export * comparing new and old * comparison improvement * post to github * fix coverage text * refactor to main from current-coverage * formatting changes * fix missing content * small tweaking * showing the Warning to make sure * formatting * remove + * checking to see if the error shows via echo * revert the change to error * separate to a new file * comment the actual test for now * prep node deps * only run certain things on main * trying cache-hit * real trying cache-hit * testing to make sure cache-run-id runs * save-always true * save-always deprecated * let's try different strategy * add key * restore-key adding a - * only perform on `main` * only run on main or if its a PR * coverage_threshold * remove comments * add total * removing unneeded comments * calculate total * run test in `release-*` only * making sure that only PR will run * only do more steps if upload-coverage successful * trying thollander/actions-comment-pull-request * using diff way to comment. * comment on how things work * testing to trigger warning and see if comment is updated vs new comment * omit echo messages * see if giving github token would work. * wrong use of param * try without github token * adding a very simple change to see where it lands * using cache hit instead. * creating the cache again. how did i lose it? * revert back * cache-hit might be off * debug * debug with failing cache restoration * check for run-id.txt instead * all into action * missing " * remove unneeded actions * change threshold to 0.5 * relative time * skeptical about date formatting * revert back to the threshold trigger * below 80% total coverage threshold * only show one error/warning at time. * testing if the coverage drop below 80 * debug output * add Reset Test Coverage label use * try using contains vs direct comparsion * remove the label checker * temp change * ooops * revert back * let's post before exiting * consistency * total coverage threshold reset to 80% --- .github/actions/test-coverage/action.yml | 131 +++++++++++++++++++++++ .github/actions/test/action.yaml | 4 + .github/workflows/ci.yml | 6 ++ jest.config.js | 2 +- scripts/compare-coverage.sh | 75 +++++++++++++ scripts/read-coverage.sh | 37 +++++++ 6 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 .github/actions/test-coverage/action.yml create mode 100755 scripts/compare-coverage.sh create mode 100755 scripts/read-coverage.sh diff --git a/.github/actions/test-coverage/action.yml b/.github/actions/test-coverage/action.yml new file mode 100644 index 000000000..3a90bed64 --- /dev/null +++ b/.github/actions/test-coverage/action.yml @@ -0,0 +1,131 @@ +# This action is used to test the coverage of the mobile repo +# It will download the coverage result from the main branch and compare it with the current branch +# If the coverage is lower than the main branch (1% or more), it will post a warning along with +# the coverage report to the PR. +# If this action is run on the main branch, it will upload the coverage result to the main branch +# It will also generate a run id and cache it, so that the next time the action is run in the PR, +# it will use the cached run id to download the coverage result from the main branch + +name: test-coverage +description: Test coverage tracking for mobile repo + +inputs: + github_token: + description: The token to use to download the coverage result + required: true + run_id: + description: The run id to use to download the coverage result + required: true + +runs: + using: composite + steps: + - name: ci/prepare-node-deps + uses: ./.github/actions/prepare-node-deps + + - name: ci/get-last-run-id + if: github.event_name == 'pull_request' + id: get-last-run-id + uses: actions/cache/restore@0c907a75c2c80ebcb7f088228285e798b750cf8f + continue-on-error: true + with: + path: run-id.txt + key: last-run-id-${{ inputs.run_id }} + restore-keys: | + last-run-id- + + - name: ci/set-pr-condition + if: github.event_name == 'pull_request' + shell: bash + run: | + echo "::group::set-pr-condition" + if [ -f "run-id.txt" ]; then + echo "IS_PR_WITH_CACHE=true" >> $GITHUB_ENV + echo "LAST_RUN_ID=$(cat run-id.txt)" >> $GITHUB_ENV + fi + echo "::endgroup::" + + - name: ci/download-main-coverage + if: env.IS_PR_WITH_CACHE == 'true' + uses: actions/download-artifact@v4 + with: + name: test-coverage-result-${{ env.LAST_RUN_ID }} + path: main-coverage/ + github-token: ${{ inputs.github_token }} + run-id: ${{ env.LAST_RUN_ID }} + + - name: ci/read-coverage + if: env.IS_PR_WITH_CACHE == 'true' + shell: bash + run: | + echo "::group::read-coverage" + ./scripts/read-coverage.sh ./main-coverage/coverage-summary.json + echo "::endgroup::" + + - name: ci/run-tests-with-coverage + shell: bash + run: | + echo "::group::run-tests" + npm run test:coverage + echo "::endgroup::" + + - name: ci/compare-coverage + if: env.IS_PR_WITH_CACHE == 'true' + id: compare-coverage + shell: bash + run: | + echo "::group::compare-coverage" + output=$(./scripts/compare-coverage.sh \ + ./main-coverage \ + ./coverage \ + ${{ github.event.pull_request.number }} \ + ${{ inputs.github_token }}) + echo "report<> $GITHUB_ENV + echo "$output" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "::endgroup::" + + - name: ci/post-coverage-report + if: env.IS_PR_WITH_CACHE == 'true' + uses: thollander/actions-comment-pull-request@v3 + with: + message: ${{ env.report }} + comment-tag: coverage-report + create-if-not-exists: true + + - name: ci/exit-on-test-coverage-failure + if: env.IS_PR_WITH_CACHE == 'true' + shell: bash + run: | + echo "::group::exit-on-test-coverage-failure" + exit ${{ steps.compare-coverage.outputs.status }} + echo "::endgroup::" + + - name: ci/upload-coverage + if: github.ref_name == 'main' + id: upload-coverage + uses: actions/upload-artifact@v4 + with: + name: test-coverage-result-${{ inputs.run_id }} + path: coverage/coverage-summary.json + overwrite: true + + - name: ci/set-upload-success + if: github.ref_name == 'main' && steps.upload-coverage.outcome == 'success' + shell: bash + run: echo "UPLOAD_SUCCESS=true" >> $GITHUB_ENV + + - name: ci/generate-run-id-file + if: env.UPLOAD_SUCCESS == 'true' + shell: bash + run: | + echo "::group::generate-last-run-id" + echo "${{ inputs.run_id }}" > run-id.txt + echo "::endgroup::" + + - name: ci/cache-run-id-file + if: env.UPLOAD_SUCCESS == 'true' + uses: actions/cache/save@0c907a75c2c80ebcb7f088228285e798b750cf8f + with: + path: run-id.txt + key: last-run-id-${{ inputs.run_id }} diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml index 4ee3d61c3..2ea363324 100644 --- a/.github/actions/test/action.yaml +++ b/.github/actions/test/action.yaml @@ -13,12 +13,16 @@ runs: echo "::group::check-styles" npm run check echo "::endgroup::" + - name: ci/run-tests + # main and PR will run test:coverage + if: startsWith(github.ref_name, 'release-') shell: bash run: | echo "::group::run-tests" npm test echo "::endgroup::" + - name: ci/check-i18n shell: bash run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a4e84af4..0622ffc04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,3 +19,9 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: ci/test uses: ./.github/actions/test + - name: ci/test-coverage + if: github.event_name == 'pull_request' || github.ref_name == 'main' + uses: ./.github/actions/test-coverage + with: + github_token: ${{ secrets.MM_MOBILE_GITHUB_TOKEN }} + run_id: ${{ github.run_id }} diff --git a/jest.config.js b/jest.config.js index 9ce77697b..213b97aad 100644 --- a/jest.config.js +++ b/jest.config.js @@ -16,7 +16,7 @@ module.exports = { clearMocks: true, setupFilesAfterEnv: ['/test/setup.ts'], collectCoverageFrom: ['app/**/*.{js,jsx,ts,tsx}'], - coverageReporters: ['lcov', 'text-summary'], + coverageReporters: ['lcov', 'text-summary', 'json-summary'], testPathIgnorePatterns: ['/node_modules/'], coveragePathIgnorePatterns: ['/node_modules/', '/components/', '/screens/'], transformIgnorePatterns: [ diff --git a/scripts/compare-coverage.sh b/scripts/compare-coverage.sh new file mode 100755 index 000000000..a33a5c5bf --- /dev/null +++ b/scripts/compare-coverage.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +COVERAGE_THRESHOLD=0.5 +PR_COVERAGE_THRESHOLD=80.0 +MAIN_COVERAGE_FILE="$1/coverage-summary.json" +RECENT_COVERAGE_FILE="$2/coverage-summary.json" +PR_NUMBER="$3" +GITHUB_TOKEN="$4" + +if [ ! -f "$MAIN_COVERAGE_FILE" ] || [ ! -f "$RECENT_COVERAGE_FILE" ]; then + echo "One or both coverage files not found" + exit 0 +fi + +COMMENT_BODY="### Coverage Comparison Report +Generated on $(date '+%B %d, %Y at %H:%M:%S UTC') + +\`\`\` ++-----------------+------------+------------+-----------+ +| Metric | Main | This PR | Diff | ++-----------------+------------+------------+-----------+" + +HAS_DECREASE=0 +BELOW_THRESHOLD=0 + +# Calculate average total coverage since the summary doesn't provide an overall total +main_total=0 +pr_total=0 +metric_count=0 + +for metric in lines statements branches functions; do + main=$(jq ".total.${metric}.pct" "$MAIN_COVERAGE_FILE") + pr=$(jq ".total.${metric}.pct" "$RECENT_COVERAGE_FILE") + diff=$(echo "$pr - $main" | bc) + + main_total=$(echo "$main_total + $main" | bc) + pr_total=$(echo "$pr_total + $pr" | bc) + metric_count=$((metric_count + 1)) + + row=$(printf "| %-15s | %9.2f%% | %9.2f%% | %8.2f%% |" "${metric^}" "$main" "$pr" "$diff") + COMMENT_BODY+=$'\n'"$row" + + if (( $(echo "$diff < -$COVERAGE_THRESHOLD" | bc -l) )); then + # Write error messages to stderr instead of stdout, since we don't want them to be shown in the PR comment + echo "::error::${metric^} coverage has decreased by more than ${COVERAGE_THRESHOLD}% ($diff%)" >&2 + HAS_DECREASE=1 + fi +done + +COMMENT_BODY+=$'\n'"+-----------------+------------+------------+-----------+" + +main_avg=$(echo "scale=2; $main_total / $metric_count" | bc) +pr_avg=$(echo "scale=2; $pr_total / $metric_count" | bc) +total_diff=$(echo "$pr_avg - $main_avg" | bc) + +if (( $(echo "$pr_avg < $PR_COVERAGE_THRESHOLD" | bc -l) )); then + echo "::error::Total coverage ($pr_avg%) is below the minimum required coverage of ${PR_COVERAGE_THRESHOLD}%" >&2 + BELOW_THRESHOLD=1 + HAS_DECREASE=1 +fi + +row=$(printf "| %-15s | %9.2f%% | %9.2f%% | %8.2f%% |" "Total" "$main_avg" "$pr_avg" "$total_diff") +COMMENT_BODY+=$'\n'"$row" + +COMMENT_BODY+=$'\n'"+-----------------+------------+------------+-----------+ +\`\`\`" + +if [ "$BELOW_THRESHOLD" -eq 1 ]; then + COMMENT_BODY+=$'\n\n'"🚨 **Error:** Total coverage ($pr_avg%) is below the minimum required coverage of ${PR_COVERAGE_THRESHOLD}%" +elif [ "$HAS_DECREASE" -eq 1 ]; then + COMMENT_BODY+=$'\n\n'"⚠️ **Warning:** One or more coverage metrics have decreased by more than ${COVERAGE_THRESHOLD}%" +fi + +echo "$COMMENT_BODY" +echo "status=$BELOW_THRESHOLD" >> $GITHUB_OUTPUT diff --git a/scripts/read-coverage.sh b/scripts/read-coverage.sh new file mode 100755 index 000000000..875879760 --- /dev/null +++ b/scripts/read-coverage.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e # Exit on any error + +# Check if an argument is provided +if [ -z "$1" ]; then + echo "❌ Error: No coverage file provided." + echo "Usage: $0 " + exit 1 +fi + +COVERAGE_FILE="$1" + +# Check if the coverage file exists +if [ ! -f "$COVERAGE_FILE" ]; then + echo "❌ Error: Coverage summary file not found: $COVERAGE_FILE" + exit 1 +fi + +# Extract coverage values from JSON +BRANCHES=$(jq '.total.branches.pct' $COVERAGE_FILE) +FUNCTIONS=$(jq '.total.functions.pct' $COVERAGE_FILE) +LINES=$(jq '.total.lines.pct' $COVERAGE_FILE) +STATEMENTS=$(jq '.total.statements.pct' $COVERAGE_FILE) + +# Print extracted values +echo "📊 Extracted Coverage Values:" +echo " - Branches: $BRANCHES%" +echo " - Functions: $FUNCTIONS%" +echo " - Lines: $LINES%" +echo " - Statements: $STATEMENTS%" + +# Export values for GitHub Actions (if needed) +echo "BRANCHES=$BRANCHES" >> $GITHUB_ENV +echo "FUNCTIONS=$FUNCTIONS" >> $GITHUB_ENV +echo "LINES=$LINES" >> $GITHUB_ENV +echo "STATEMENTS=$STATEMENTS" >> $GITHUB_ENV