diff --git a/.eslintignore b/.eslintignore index 91f5433596..54824be5c7 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,4 +6,5 @@ packages/server/coverage packages/server/client packages/builder/.routify packages/builder/cypress/support/queryLevelTransformerFunction.js -packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js \ No newline at end of file +packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js +packages/builder/cypress/reports \ No newline at end of file diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 14aee034d3..a53550a89f 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -38,19 +38,15 @@ jobs: env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - # TODO: upload recordings to s3 - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: eu-west-1 - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 + - uses: actions/upload-artifact@v3 with: - webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }} - content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.outputs.dashboardUrl }}" - embed-title: ${{ steps.cypress.outcome }} - embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }} + name: Test Reports + path: packages/cypress/reports/mocha + - name: Cypress Discord Notify + run: yarn test:e2e:ci:notify + env: + CYPRESS_WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }} + CYPRESS_OUTCOME: ${{ steps.cypress.outcome }} + CYPRESS_DASHBOARD_URL: ${{ steps.cypress.outputs.dashboardUrl }} + GITHUB_RUN_URL: ${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }} diff --git a/.gitignore b/.gitignore index 7d09f0a2ba..03d77c5477 100644 --- a/.gitignore +++ b/.gitignore @@ -97,5 +97,7 @@ hosting/proxy/.generated-nginx.prod.conf bin/ hosting/.generated* -packages/builder/cypress.env.json -stats.html +packages/builder/cypress.env.json +packages/builder/cypress/reports +stats.html + diff --git a/package.json b/package.json index fb6d9da990..d8dbea524b 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "test:e2e": "lerna run cy:test --stream", "test:e2e:ci": "lerna run cy:ci --stream", "test:e2e:ci:record": "lerna run cy:ci:record --stream", + "test:e2e:ci:report": "lerna run cy:ci:report", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", diff --git a/packages/builder/package.json b/packages/builder/package.json index 8987a56a47..e031947f02 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -13,11 +13,12 @@ "cy:setup:ci": "node ./cypress/setup.js", "cy:open": "cypress open", "cy:run": "cypress run", - "cy:run:ci": "cypress run --headed --browser chrome", + "cy:run:ci": "cypress run --headed --browser chrome --spec cypress/integration/createApp.spec.js", "cy:run:ci:record": "xvfb-run cypress run --headed --browser chrome --record", "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run", "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci", "cy:ci:record": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci:record", + "cy:ci:report": "node scripts/cypressResultsWebhook", "cy:debug": "start-server-and-test cy:setup http://localhost:4100/builder cy:open", "cy:debug:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:open" }, @@ -99,7 +100,6 @@ "babel-jest": "^26.6.3", "cypress": "^9.3.1", "cypress-multi-reporters": "^1.6.0", - "cypress-slack-reporter": "^1.5.0", "cypress-terminal-report": "^1.4.1", "identity-obj-proxy": "^3.0.0", "jest": "^26.6.3", diff --git a/packages/builder/reporterConfig.json b/packages/builder/reporterConfig.json index 30126987fc..a3b713a3cd 100644 --- a/packages/builder/reporterConfig.json +++ b/packages/builder/reporterConfig.json @@ -4,7 +4,7 @@ "reportDir": "cypress/reports/mocha", "quiet": true, "overwrite": false, - "html": false, + "html": true, "json": true } } \ No newline at end of file diff --git a/packages/builder/scripts/cypressResultsWebhook.js b/packages/builder/scripts/cypressResultsWebhook.js new file mode 100644 index 0000000000..8e1484ca04 --- /dev/null +++ b/packages/builder/scripts/cypressResultsWebhook.js @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +const fetch = require("node-fetch") +const fs = require("fs") +const path = require("path") + +const WEBHOOK_URL = process.env.CYPRESS_WEBHOOK_URL +const OUTCOME = process.env.CYPRESS_OUTCOME +const DASHBOARD_URL = process.env.CYPRESS_DASHBOARD_URL +const GIT_SHA = process.env.GITHUB_SHA +const GITHUB_ACTIONS_RUN_URL = process.env.GITHUB_ACTIONS_RUN_URL + +// read the report file +const REPORT_PATH = path.resolve( + __dirname, + "..", + "cypress", + "reports", + "mocha", + "mochawesome.json" +) +const testReport = JSON.parse(fs.readFileSync(REPORT_PATH, "utf-8")) + +const { + suites, + tests, + passes, + pending, + failures, + duration, + passPercent, + skipped, +} = testReport.stats + +async function discordCypressResultsNotification() { + const options = { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + content: `**Nightly Tests Status**: ${OUTCOME}`, + embeds: [ + { + title: "Budi QA Bot", + description: `Nightly Tests`, + url: GITHUB_ACTIONS_RUN_URL, + color: OUTCOME === "success" ? 3066993 : 15548997, + timestamp: new Date(), + footer: { + icon_url: "http://bbui.budibase.com/budibase-logo.png", + text: "Budibase QA Bot", + }, + thumbnail: { + url: "http://bbui.budibase.com/budibase-logo.png", + }, + author: { + name: "Budibase QA Bot", + url: "https://discordapp.com", + icon_url: "http://bbui.budibase.com/budibase-logo.png", + }, + fields: [ + { + name: "Commit", + value: GIT_SHA, + }, + { + name: "Cypress Dashboard URL", + value: DASHBOARD_URL || "None Supplied", + }, + { + name: "Github Actions Run URL", + value: GITHUB_ACTIONS_RUN_URL, + }, + { + name: "Test Suites", + value: suites, + }, + { + name: "Tests", + value: tests, + }, + { + name: "Passed", + value: passes, + }, + { + name: "Pending", + value: pending, + }, + { + name: "Skipped", + value: skipped, + }, + { + name: "Failures", + value: failures, + }, + { + name: "Duration", + value: `${duration / 1000} Seconds`, + }, + { + name: "Pass Percentage", + value: passPercent, + }, + ], + }, + ], + }), + } + const response = await fetch(WEBHOOK_URL, options) + + if (response.status >= 400) { + const text = await response.text() + console.error( + `Error sending discord webhook. \nStatus: ${response.status}. \nResponse Body: ${text}. \nRequest Body: ${options.body}` + ) + } +} + +async function run() { + await discordCypressResultsNotification() +} + +run()