From e7d597f70eecdc6b72ba06f5f2e6913a91a7cbc4 Mon Sep 17 00:00:00 2001 From: Rahim Rahman Date: Mon, 14 Oct 2024 07:56:52 -0600 Subject: [PATCH] test: mitmproxy + low bandwidth + automated QA using detox (#8202) * experimental github action * start mtmproxy then kill it * suspend test for now * dont need to be in a PR to trigger * trying again with mitmdump * where is .mitmproxy folder? * run with +e * not sure why exit code 1 * if this is a better path * let's try killing using trap? * trying a different path * maybe using ~ would work? * looking at output to networksetup command * minor mistake * throttling bandwidth * lets try with sudo * missing sudo * using mitmdump * remove spaces * tried diff path * change to the proper file * upload artifacts * install cert * added cert to simulator * getting simulator UDID * proper scoped var * using github outputs * getenv doesn't have IOS_SIMULATOR_UDID * using echo command * maybe forgot to scope outputs * lets run the command directly * changed to iPhone 14 * let me see all the devices * try something new * rearrange my steps to get cert * getenv booted * remove too many outputs * using mm mobile test server * tie it all together with detox test * removed dependency for another job * put dependency on a job * simplify process even more * add github-token on download artifact * add run-id, hopefully that's what needed to download * incorrect speed for download * update to the correct dummynet syntax * proper mitmdump flow report * update allow-hosts * commenting out mitmdump for now * back to rahim-experiments * overly commented out * cacert not --ca-cert * maybe i allow the wrong host? * maybe i just forgotr to kill mitmdump after I'm done? * using pm2 * reinstate jobs. and longer timeout for detox * need sudo * recording videos and limiting to 1 test * just want to run 1 test. * back to experimenting * using wget * test using chrome * using chrome and add delay * ok test again with detox * go back to testing all server_login * compare booted simulators * few more experiments * no \ * using github_outputs * using fromJSON * too many $ * can i do this another way? * two runs to get the it assigned * try again * . * trying to access devices array * try again * using jq * keep it simple * might not need outputs * forgot pipe * will this work? * will first element of an array works * see if we got the udid * did we get it wrong? * will this work? * let's assign? * we have to put into string * ok 1 more * will it have quotes? * echo in an echo * is this working? * let's hope this works. * it's actually using 17.4 iPhone 14 not 17.0 * broke because i add & * removed failed step * lowering ping and going back to 10 parallel test * double-up ONE_SEC to see if tests will pass * why curl failed after? * getting more data from failures * let's get more logs with pm and mitmdump * more logging and improvement * silly error! * problem with throttling, let's figure out where we can improve * increase timeout for low-bandwidth * increased delay + run all tests * lets see if this will be better for throttling * try set +e to see if it will return failure * check disk space. * deleting zip and trash to free up space. * try to do pfctl twice to see if that would help * making bandwidth-throttling a composite action * would 5 seconds make a diff? * add shell * throttle bandwidth experiment * fix quick problem * 5s sleep, and throttling at the right time * the throttling was working better earlier. * add a flush in between * no exit, see if it works well * trying continue-on-error * resetting with continue-on-error vs checking for if steps before worked * adding flush before throttling to see if it makes a diff * put it back where we think is the best spot * change timeout to 60 and some cleanup * forgot a simple $ * move network throttling again to see if this is better? * rearranging * move throttling back to where it used to work well making bandwidth-throttling action taking inputs * forgot shell * inputs not input, and continue-on-error * getting host from site_1_url * update the sed script * move more steps into composite action file * add shell * add more shell * action cleanup * see if our reset network setting works * delete unused files * more clean up * wrong indentation * wrong context * see if not having low bandwidth enabled would cause stuff to break * will it fail on non-existent step name? * things are failing after proxy started * start using low_bandwidth_mode input * incorrect use of curly bracket * low_bandwidth_mode vs low_bandwidth * correcting the use of curly bracket in if check * low_bandwidth_mode correction * why would https:// makes a diff? * just going to start pm2 after stopping * move things around again * some more re-arranging * exit 1 * rearranged too much * clean up * start-proxy action * incorrect variable name * use test_server_url instead * reverting back to original * use pull_request * using env var prepend * using existing artifact * not in "" * not going to prepend anymore * incorrectly removed input instead of env * close to final * using original e2e-detox-template * timeout-minutes need to be higher for low_bandwidth. * using label instead of push to branch * add check-label * remove unneeded test * renamed ci.yml back to original * different way to remove label remove experiment files reinstate ci.yml * remove experiment file * see label again * output labels * another experiment * test again * fix issue when checking for low bandwidth * revert changes to package.json in detox/ * changed download to upload in the description * changes based on review * add github.event.label.name to the group "id" per @mvitale1989 suggestion --------- Co-authored-by: Mattermost Build --- .../actions/bandwidth-throttling/action.yml | 71 ++++++++++++ .../actions/prepare-low-bandwidth/action.yml | 106 ++++++++++++++++++ .github/actions/start-proxy/action.yml | 36 ++++++ .github/workflows/e2e-detox-pr.yml | 30 +++-- .github/workflows/e2e-detox-template.yml | 48 +++++++- detox/e2e/support/utils/index.ts | 3 +- scripts/mitmdump-flow-parsing.py | 52 +++++++++ 7 files changed, 332 insertions(+), 14 deletions(-) create mode 100644 .github/actions/bandwidth-throttling/action.yml create mode 100644 .github/actions/prepare-low-bandwidth/action.yml create mode 100644 .github/actions/start-proxy/action.yml create mode 100644 scripts/mitmdump-flow-parsing.py diff --git a/.github/actions/bandwidth-throttling/action.yml b/.github/actions/bandwidth-throttling/action.yml new file mode 100644 index 000000000..70cb4b8c7 --- /dev/null +++ b/.github/actions/bandwidth-throttling/action.yml @@ -0,0 +1,71 @@ +name: bandwidth-throttling +description: Action to throttle the bandwidth on MacOS runner + +inputs: + test_server_host: + description: The host of the test server, no protocol + required: true + download_speed: + description: The download speed limit (in Kbit/s) + required: false + default: "3300" + upload_speed: + description: The upload speed limit (in Kbit/s) + required: false + default: "3300" + latency: + description: The latency (in ms) each way + required: false + default: "500" + disable: + description: Disable throttling + required: false + default: "false" + +runs: + using: composite + steps: + - name: disable first + if: ${{ inputs.disable == 'true' }} + shell: bash + continue-on-error: true + run: | + sudo pfctl -d + + sleep 2; + + - name: throttle bandwidth down + shell: bash + run: | + # reset pf and dnctl + sudo dnctl -q flush + sudo dnctl -q pipe flush + sudo pfctl -f /etc/pf.conf + sudo pfctl -E + + sleep 2; + + sudo pfctl -d + sudo dnctl -q flush + sudo dnctl -q pipe flush + + + echo "dummynet in from ${{ inputs.test_server_host }} to ! 127.0.0.1 pipe 1 + dummynet out from ! 127.0.0.1 to ${{ inputs.test_server_host }} pipe 2" | sudo pfctl -f - + + # pipe 1 is download + sudo dnctl pipe 1 config bw ${{ inputs.download_speed }}Kbit/s delay ${{ inputs.latency }}ms + + # pipe 2 is upload + sudo dnctl pipe 2 config bw ${{ inputs.upload_speed }}Kbit/s delay ${{ inputs.latency }}ms + + sleep 5; + + sudo pfctl -E + + sleep 5; + + - name: test curl after throttling + shell: bash + run: | + curl -o /dev/null -m 20 --retry 2 -s -w 'Total: %{time_total}s\n' 'https://${{ inputs.test_server_host }}/api/v4/system/ping?get_server_status=true' diff --git a/.github/actions/prepare-low-bandwidth/action.yml b/.github/actions/prepare-low-bandwidth/action.yml new file mode 100644 index 000000000..b39b77c27 --- /dev/null +++ b/.github/actions/prepare-low-bandwidth/action.yml @@ -0,0 +1,106 @@ +name: Prepare Low Bandwidth Environment (MacOS & iOS Simulators only) +description: prepare any workflow for low bandwidth testing + +inputs: + test_server_url: + description: The URL of the test server + required: true + device_name: + description: The iOS simulator name + required: true + download_speed: + description: The download speed limit (in Kbit/s) + required: false + default: "3300" + upload_speed: + description: The upload speed limit (in Kbit/s) + required: false + default: "3300" + latency: + description: The latency (in ms) each way + required: false + default: "500" + + +runs: + using: composite + steps: + - name: delete the zip file and trash (to free up space) + shell: bash + run: | + rm -rf mobile-artifacts/*.zip + sudo rm -rf ~/.Trash/* + + - name: check disk space + shell: bash + run: df -h + + - name: remove protocol from SITE_1_URL + id: remove-protocol + shell: bash + run: | + echo "SITE_1_HOST=${{ inputs.test_server_url }}" | sed -e 's/http:\/\///g' -e 's/https:\/\///g' >> ${GITHUB_OUTPUT} + + - name: Throttle Bandwidth 1 + id: throttle-bandwidth-1 + continue-on-error: true + uses: ./.github/actions/bandwidth-throttling + with: + test_server_host: ${{ steps.remove-protocol.outputs.SITE_1_HOST }} + download_speed: ${{ inputs.download_speed }} + upload_speed: ${{ inputs.upload_speed }} + latency: ${{ inputs.latency }} + + - name: Throttle Bandwidth 2 + if: steps.throttle-bandwidth-1.outcome != 'success' + id: throttle-bandwidth-2 + uses: ./.github/actions/bandwidth-throttling + with: + test_server_host: ${{ steps.remove-protocol.outputs.SITE_1_HOST }} + download_speed: ${{ inputs.download_speed}} + upload_speed: ${{ inputs.upload_speed }} + latency: ${{ inputs.latency }} + disable: "true" + + - name: Install mitmproxy & pm2 (process manager) + id: install-mitmproxy-pm2 + shell: bash + run: | + brew install mitmproxy + + npm i -g pm2 + + - name: Start mitmproxy via mitmdump and stop it (to get .mitmproxy folder) + shell: bash + run: | + pm2 start mitmdump --log /Users/runner/work/mattermost-mobile/mattermost-mobile/mitmdump.log -- --allow-hosts '${{ steps.remove-protocol.outputs.SITE_1_HOST }}' --ignore-hosts 'localhost' -s /Users/runner/work/mattermost-mobile/mattermost-mobile/scripts/mitmdump-flow-parsing.py + + sleep 5; + + pm2 stop mitmdump + + # we need to wait for mitmdump to stop so it'll produce the .mitmproxy folder + sleep 5; + + - name: Get simulator UDID + id: get-simulator-udid + shell: bash + run: | + simulator_udid=$(xcrun simctl list devices "${{ inputs.device_name }}" -j | jq '.devices' | jq '."com.apple.CoreSimulator.SimRuntime.iOS-17-4"[0]["udid"]') + echo "simulator_udid="$(echo $simulator_udid) >> ${GITHUB_OUTPUT} + + - name: install certificate + shell: bash + run: | + sudo security add-trusted-cert -d -p ssl -p basic -k /Library/Keychains/System.keychain ~/.mitmproxy/mitmproxy-ca-cert.pem + + # must boot first before adding root cert + xcrun simctl boot ${{ steps.get-simulator-udid.outputs.simulator_udid }} + + xcrun simctl keychain booted add-root-cert ~/.mitmproxy/mitmproxy-ca-cert.pem + + sleep 5; + + - name: show me booted simulators + shell: bash + run: xcrun simctl list devices booted | grep Booted diff --git a/.github/actions/start-proxy/action.yml b/.github/actions/start-proxy/action.yml new file mode 100644 index 000000000..452572d2a --- /dev/null +++ b/.github/actions/start-proxy/action.yml @@ -0,0 +1,36 @@ +name: Start Proxy +description: Action to throttle the bandwidth on MacOS runner + +inputs: + test_server_url: + description: The host of the test server, no protocol + required: true + +runs: + using: composite + steps: + + - name: restart mitmdump + shell: bash + run: | + pm2 restart mitmdump + + sleep 5; + + - name: start proxy + shell: bash + run: | + networksetup -setwebproxy Ethernet "127.0.0.1" "8080" + networksetup -setsecurewebproxy Ethernet "127.0.0.1" "8080" + + sleep 5; + + networksetup -getwebproxy Ethernet + networksetup -getsecurewebproxy Ethernet + + - name: test curl and direct it into proxy + shell: bash + run: | + curl -o /dev/null -m 20 -s -w 'Total: %{time_total}s\n' '${{ inputs.test_server_url }}/api/v4/system/ping?get_server_status=true' + + curl --proxy "127.0.0.1:8080" -o /dev/null -m 20 -s -w 'Total: %{time_total}s\n' '${{ inputs.test_server_url }}/api/v4/system/ping?get_server_status=true' diff --git a/.github/workflows/e2e-detox-pr.yml b/.github/workflows/e2e-detox-pr.yml index a36299dad..41429ecb2 100644 --- a/.github/workflows/e2e-detox-pr.yml +++ b/.github/workflows/e2e-detox-pr.yml @@ -1,3 +1,5 @@ +# Can be used to run Detox E2E tests on pull requests for the Mattermost mobile app with low bandwidth +# by using 'E2E iOS tests for PR (LBW 1)' instead. name: Detox E2E Tests PR on: @@ -7,10 +9,14 @@ on: types: - labeled +concurrency: + group: "${{ github.workflow }}-${{ github.event.pull_request.number }}-${{ github.event.label.name }}" + cancel-in-progress: true + jobs: update-initial-status: runs-on: ubuntu-22.04 - if: contains(github.event.pull_request.labels.*.name, 'E2E iOS tests for PR') + if: contains(github.event.label.name, 'E2E iOS tests for PR') steps: - uses: mattermost/actions/delivery/update-commit-status@main env: @@ -57,6 +63,7 @@ jobs: run-ios-tests: true run-type: "PR" MOBILE_VERSION: ${{ github.event.pull_request.head.sha }} + low_bandwidth_mode: ${{ contains(github.event.label.name,'LBW') && true || false }} secrets: inherit update-final-status: @@ -87,14 +94,13 @@ jobs: script: | const iosLabel = 'E2E iOS tests for PR'; const androidLabel = 'E2E Android tests for PR'; - const labels = context.payload.pull_request.labels.map(label => label.name); - - if (labels.includes(iosLabel)) { - github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: iosLabel, - }); - } - + context.payload.pull_request.labels.forEach(label => { + if (label.name.includes(iosLabel)) { + github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: label.name, + }); + } + }); diff --git a/.github/workflows/e2e-detox-template.yml b/.github/workflows/e2e-detox-template.yml index bf3c4102f..659f793e2 100644 --- a/.github/workflows/e2e-detox-template.yml +++ b/.github/workflows/e2e-detox-template.yml @@ -48,6 +48,11 @@ on: required: false type: string default: "iOS 17.0" + low_bandwidth_mode: + description: "Enable low bandwidth mode" + required: false + type: boolean + default: false outputs: STATUS: value: ${{ jobs.generate-report.outputs.STATUS }} @@ -123,7 +128,7 @@ jobs: if: ${{ inputs.run-ios-tests }} runs-on: macos-14 continue-on-error: true - timeout-minutes: 40 + timeout-minutes: ${{ inputs.low_bandwidth_mode && 80 || 40 }} env: IOS: true needs: @@ -154,12 +159,31 @@ jobs: - name: Unzip iOS Simulator Build run: unzip -o mobile-artifacts/*.zip -d mobile-artifacts/ + - name: Prepare Low Bandwidth Environment + id: prepare-low-bandwidth + uses: ./.github/actions/prepare-low-bandwidth + if: ${{ inputs.low_bandwidth_mode }} + with: + test_server_url: ${{ env.SITE_1_URL }} + device_name: ${{ env.DEVICE_NAME }} + # all these value should be configurable + download_speed: "3300" + upload_speed: "3300" + latency: "500" + - name: Start React Native Metro Server run: npm run start & - name: Install Detox Dependencies run: cd detox && npm i + - name: Start Proxy + if: ${{ inputs.low_bandwidth_mode }} + id: start-proxy + uses: ./.github/actions/start-proxy + with: + test_server_url: ${{ env.SITE_1_URL }} + - name: Run Detox E2E Tests continue-on-error: true # Label might have been removed manually run: | @@ -169,6 +193,28 @@ jobs: env: DETOX_DEVICE_TYPE: ${{ env.DEVICE_NAME }} DETOX_OS_VERSION: ${{ env.DEVICE_OS_VERSION }} + LOW_BANDWIDTH_MODE: ${{ inputs.low_bandwidth_mode }} + + + - name: reset network settings + if: ${{ inputs.low_bandwidth_mode || failure() }} + run: | + networksetup -setwebproxystate Ethernet "off" + networksetup -setsecurewebproxystate Ethernet "off" + + if (sudo pfctl -q -sa | grep 'Status: Enabled') then sudo pfctl -d; fi + if (command -v pm2 &> /dev/null) then pm2 stop mitmdump; fi + + sleep 5; + + - name: Upload mitmdump Flow Output + if: ${{ inputs.low_bandwidth_mode }} + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: ios-mitmdump-flow-output-${{ needs.generate-specs.outputs.workflow_hash }}-${{ matrix.runId }} + path: | + /Users/runner/work/mattermost-mobile/mattermost-mobile/flow-output.csv + /Users/runner/work/mattermost-mobile/mattermost-mobile/mitmdump.log - name: Upload iOS Test Report if: always() diff --git a/detox/e2e/support/utils/index.ts b/detox/e2e/support/utils/index.ts index cba31c95c..6612b8d81 100644 --- a/detox/e2e/support/utils/index.ts +++ b/detox/e2e/support/utils/index.ts @@ -62,7 +62,7 @@ export const getAdminAccount = () => { }; }; -const SECOND = 1000; +const SECOND = 1000 * (process.env.LOW_BANDWIDTH_MODE === 'true' ? 5 : 1); const MINUTE = 60 * 1000; export const timeouts = { @@ -76,3 +76,4 @@ export const timeouts = { TWO_MIN: MINUTE * 2, FOUR_MIN: MINUTE * 4, }; + diff --git a/scripts/mitmdump-flow-parsing.py b/scripts/mitmdump-flow-parsing.py new file mode 100644 index 000000000..2ead3f4da --- /dev/null +++ b/scripts/mitmdump-flow-parsing.py @@ -0,0 +1,52 @@ +from mitmproxy.net.http.http1.assemble import assemble_request, assemble_response +from datetime import datetime +import math +from urllib.parse import urlsplit + +# Use a context manager to handle file operations +f = open('./flow-output.csv', 'w') + +def response(flow): + # Convert timestamps to milliseconds + req_ts = math.floor(flow.request.timestamp_start * 1000) + req_te = math.floor(flow.request.timestamp_end * 1000) + resp_ts = math.floor(flow.response.timestamp_start * 1000) + resp_te = math.floor(flow.response.timestamp_end * 1000) + + # Calculate total times + request_total_time = req_te - req_ts + response_total_time = resp_te - resp_ts + total_time = resp_te - req_ts + + # Convert timestamps to human-readable dates + req_ts_human = datetime.fromtimestamp(req_ts / 1000).strftime('%Y-%m-%d %H:%M:%S.%f') + req_te_human = datetime.fromtimestamp(req_te / 1000).strftime('%Y-%m-%d %H:%M:%S.%f') + resp_ts_human = datetime.fromtimestamp(resp_ts / 1000).strftime('%Y-%m-%d %H:%M:%S.%f') + resp_te_human = datetime.fromtimestamp(resp_te / 1000).strftime('%Y-%m-%d %H:%M:%S.%f') + + # Parse the URL to separate path and query string + url_parts = urlsplit(flow.request.path) + path = url_parts.path + query = url_parts.query + + # Format data as a comma-separated string + data = ( + f"{req_ts_human}," + f"{req_te_human}," + f"{request_total_time}," + f"{flow.request.method}," + f"{flow.request.host}," + f"{path}," + f"{query}," + f"{flow.response.status_code}," + f"{resp_ts_human}," + f"{resp_te_human}," + f"{response_total_time}," + f"{total_time}\n" + ) + + # need to figure out how to get the content-length header + # f"{flow.response.headers['Content-Length']}\n" + + # Write the formatted string to the file + f.write(data)