diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index d6e0432e83..d63596f08f 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -25,7 +25,7 @@ env: BASE_BRANCH: ${{ github.event.pull_request.base.ref}} PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} - USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }} + ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }} IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }} jobs: @@ -72,7 +72,7 @@ jobs: # Check the types of the projects built via esbuild - name: Check types run: | - if ${{ env.USE_NX_AFFECTED }}; then + if ${{ env.ONLY_AFFECTED_TASKS }}; then yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server else yarn check:types --ignore @budibase/account-portal-server @@ -108,7 +108,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.2.1-sql & + docker pull budibase/couchdb:v3.2.1-sqs & docker pull redis & wait $(jobs -p) @@ -116,7 +116,7 @@ jobs: - run: yarn --frozen-lockfile - name: Test run: | - if ${{ env.USE_NX_AFFECTED }}; then + if ${{ env.ONLY_AFFECTED_TASKS }}; then yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }} else yarn test --ignore=@budibase/worker --ignore=@budibase/server @@ -140,8 +140,8 @@ jobs: - run: yarn --frozen-lockfile - name: Test worker run: | - if ${{ env.USE_NX_AFFECTED }}; then - yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }} + if ${{ env.ONLY_AFFECTED_TASKS }}; then + node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }} else yarn test --scope=@budibase/worker fi @@ -180,8 +180,8 @@ jobs: - name: Test server run: | - if ${{ env.USE_NX_AFFECTED }}; then - yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} + if ${{ env.ONLY_AFFECTED_TASKS }}; then + node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} else yarn test --scope=@budibase/server fi @@ -214,6 +214,7 @@ jobs: echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" + echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit) echo "base_commit_excluding_merges=$base_commit_excluding_merges" @@ -230,7 +231,7 @@ jobs: base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' - any_commit=$(git log --no-merges $base_commit...$pro_commit) + any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit) if [ -n "$any_commit" ]; then echo $any_commit diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 481197b26c..7ecf264add 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "bulma": "^0.9.3", - "next": "12.1.0", + "next": "14.1.1", "node-fetch": "^3.2.10", "sass": "^1.52.3", "react": "17.0.2", diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index 93e26a954d..2c36066211 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -46,10 +46,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== -"@next/env@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/env/-/env-12.1.0.tgz#73713399399b34aa5a01771fb73272b55b22c314" - integrity sha512-nrIgY6t17FQ9xxwH3jj0a6EOiQ/WDHUos35Hghtr+SWN/ntHIQ7UpuvSi0vaLzZVHQWaDupKI+liO5vANcDeTQ== +"@next/env@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac" + integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA== "@next/eslint-plugin-next@12.1.0": version "12.1.0" @@ -58,60 +58,50 @@ dependencies: glob "7.1.7" -"@next/swc-android-arm64@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-android-arm64/-/swc-android-arm64-12.1.0.tgz#865ba3a9afc204ff2bdeea49dd64d58705007a39" - integrity sha512-/280MLdZe0W03stA69iL+v6I+J1ascrQ6FrXBlXGCsGzrfMaGr7fskMa0T5AhQIVQD4nA/46QQWxG//DYuFBcA== +"@next/swc-darwin-arm64@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64" + integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ== -"@next/swc-darwin-arm64@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-12.1.0.tgz#08e8b411b8accd095009ed12efbc2f1d4d547135" - integrity sha512-R8vcXE2/iONJ1Unf5Ptqjk6LRW3bggH+8drNkkzH4FLEQkHtELhvcmJwkXcuipyQCsIakldAXhRbZmm3YN1vXg== +"@next/swc-darwin-x64@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b" + integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw== -"@next/swc-darwin-x64@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-12.1.0.tgz#fcd684497a76e8feaca88db3c394480ff0b007cd" - integrity sha512-ieAz0/J0PhmbZBB8+EA/JGdhRHBogF8BWaeqR7hwveb6SYEIJaDNQy0I+ZN8gF8hLj63bEDxJAs/cEhdnTq+ug== +"@next/swc-linux-arm64-gnu@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa" + integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg== -"@next/swc-linux-arm-gnueabihf@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.1.0.tgz#9ec6380a27938a5799aaa6035c205b3c478468a7" - integrity sha512-njUd9hpl6o6A5d08dC0cKAgXKCzm5fFtgGe6i0eko8IAdtAPbtHxtpre3VeSxdZvuGFh+hb0REySQP9T1ttkog== +"@next/swc-linux-arm64-musl@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a" + integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ== -"@next/swc-linux-arm64-gnu@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.1.0.tgz#7f4196dff1049cea479607c75b81033ae2dbd093" - integrity sha512-OqangJLkRxVxMhDtcb7Qn1xjzFA3s50EIxY7mljbSCLybU+sByPaWAHY4px97ieOlr2y4S0xdPKkQ3BCAwyo6Q== +"@next/swc-linux-x64-gnu@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528" + integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ== -"@next/swc-linux-arm64-musl@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-12.1.0.tgz#b445f767569cdc2dddee785ca495e1a88c025566" - integrity sha512-hB8cLSt4GdmOpcwRe2UzI5UWn6HHO/vLkr5OTuNvCJ5xGDwpPXelVkYW/0+C3g5axbDW2Tym4S+MQCkkH9QfWA== +"@next/swc-linux-x64-musl@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25" + integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og== -"@next/swc-linux-x64-gnu@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-12.1.0.tgz#67610e9be4fbc987de7535f1bcb17e45fe12f90e" - integrity sha512-OKO4R/digvrVuweSw/uBM4nSdyzsBV5EwkUeeG4KVpkIZEe64ZwRpnFB65bC6hGwxIBnTv5NMSnJ+0K/WmG78A== +"@next/swc-win32-arm64-msvc@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9" + integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A== -"@next/swc-linux-x64-musl@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-12.1.0.tgz#ea19a23db08a9f2e34ac30401f774cf7d1669d31" - integrity sha512-JohhgAHZvOD3rQY7tlp7NlmvtvYHBYgY0x5ZCecUT6eCCcl9lv6iV3nfu82ErkxNk1H893fqH0FUpznZ/H3pSw== +"@next/swc-win32-ia32-msvc@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2" + integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw== -"@next/swc-win32-arm64-msvc@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.1.0.tgz#eadf054fc412085659b98e145435bbba200b5283" - integrity sha512-T/3gIE6QEfKIJ4dmJk75v9hhNiYZhQYAoYm4iVo1TgcsuaKLFa+zMPh4056AHiG6n9tn2UQ1CFE8EoybEsqsSw== - -"@next/swc-win32-ia32-msvc@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.1.0.tgz#68faeae10c89f698bf9d28759172b74c9c21bda1" - integrity sha512-iwnKgHJdqhIW19H9PRPM9j55V6RdcOo6rX+5imx832BCWzkDbyomWnlzBfr6ByUYfhohb8QuH4hSGEikpPqI0Q== - -"@next/swc-win32-x64-msvc@12.1.0": - version "12.1.0" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-12.1.0.tgz#d27e7e76c87a460a4da99c5bfdb1618dcd6cd064" - integrity sha512-aBvcbMwuanDH4EMrL2TthNJy+4nP59Bimn8egqv6GHMVj0a44cU6Au4PjOhLNqEh9l+IpRGBqMTzec94UdC5xg== +"@next/swc-win32-x64-msvc@14.1.1": + version "14.1.1" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52" + integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -139,6 +129,13 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A== +"@swc/helpers@0.5.2": + version "0.5.2" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d" + integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== + dependencies: + tslib "^2.4.0" + "@types/json5@^0.0.29": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" @@ -344,6 +341,13 @@ bulma@^0.9.3: resolved "https://registry.yarnpkg.com/bulma/-/bulma-0.9.3.tgz#ddccb7436ebe3e21bf47afe01d3c43a296b70243" integrity sha512-0d7GNW1PY4ud8TWxdNcP6Cc8Bu7MxcntD/RRLGWuiw/s0a9P+XlH/6QoOIrmbj6o8WWJzJYhytiu9nFjTszk1g== +busboy@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" + integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== + dependencies: + streamsearch "^1.1.0" + call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" @@ -357,10 +361,10 @@ callsites@^3.0.0: resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -caniuse-lite@^1.0.30001283: - version "1.0.30001314" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001314.tgz#65c7f9fb7e4594fca0a333bec1d8939662377596" - integrity sha512-0zaSO+TnCHtHJIbpLroX7nsD+vYuOVjl3uzFbJO1wMVbuveJA0RK2WcQA9ZUIOiO0/ArMiMgHJLxfEZhQiC0kw== +caniuse-lite@^1.0.30001579: + version "1.0.30001640" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001640.tgz#32c467d4bf1f1a0faa63fc793c2ba81169e7652f" + integrity sha512-lA4VMpW0PSUrFnkmVuEKBUovSWKhj7puyCg8StBChgu298N1AtuF1sKWEvfDuimSEDbhlb/KqPKC3fs1HbuQUA== chalk@^4.0.0: version "4.1.2" @@ -385,6 +389,11 @@ chalk@^4.0.0: optionalDependencies: fsevents "~2.3.2" +client-only@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/client-only/-/client-only-0.0.1.tgz#38bba5d403c41ab150bff64a95c85013cf73bca1" + integrity sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA== + color-convert@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" @@ -909,6 +918,11 @@ globby@^11.0.4: merge2 "^1.4.1" slash "^3.0.0" +graceful-fs@^4.2.11: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + has-bigints@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.1.tgz#64fe6acb020673e3b78db035a5af69aa9d07b113" @@ -1221,38 +1235,38 @@ ms@^2.1.1: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -nanoid@^3.1.30: - version "3.3.1" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.1.tgz#6347a18cac88af88f58af0b3594b723d5e99bb35" - integrity sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw== +nanoid@^3.3.6: + version "3.3.7" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" + integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next@12.1.0: - version "12.1.0" - resolved "https://registry.yarnpkg.com/next/-/next-12.1.0.tgz#c33d753b644be92fc58e06e5a214f143da61dd5d" - integrity sha512-s885kWvnIlxsUFHq9UGyIyLiuD0G3BUC/xrH0CEnH5lHEWkwQcHOORgbDF0hbrW9vr/7am4ETfX4A7M6DjrE7Q== +next@14.1.1: + version "14.1.1" + resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171" + integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww== dependencies: - "@next/env" "12.1.0" - caniuse-lite "^1.0.30001283" - postcss "8.4.5" - styled-jsx "5.0.0" - use-subscription "1.5.1" + "@next/env" "14.1.1" + "@swc/helpers" "0.5.2" + busboy "1.6.0" + caniuse-lite "^1.0.30001579" + graceful-fs "^4.2.11" + postcss "8.4.31" + styled-jsx "5.1.1" optionalDependencies: - "@next/swc-android-arm64" "12.1.0" - "@next/swc-darwin-arm64" "12.1.0" - "@next/swc-darwin-x64" "12.1.0" - "@next/swc-linux-arm-gnueabihf" "12.1.0" - "@next/swc-linux-arm64-gnu" "12.1.0" - "@next/swc-linux-arm64-musl" "12.1.0" - "@next/swc-linux-x64-gnu" "12.1.0" - "@next/swc-linux-x64-musl" "12.1.0" - "@next/swc-win32-arm64-msvc" "12.1.0" - "@next/swc-win32-ia32-msvc" "12.1.0" - "@next/swc-win32-x64-msvc" "12.1.0" + "@next/swc-darwin-arm64" "14.1.1" + "@next/swc-darwin-x64" "14.1.1" + "@next/swc-linux-arm64-gnu" "14.1.1" + "@next/swc-linux-arm64-musl" "14.1.1" + "@next/swc-linux-x64-gnu" "14.1.1" + "@next/swc-linux-x64-musl" "14.1.1" + "@next/swc-win32-arm64-msvc" "14.1.1" + "@next/swc-win32-ia32-msvc" "14.1.1" + "@next/swc-win32-x64-msvc" "14.1.1" node-domexception@^1.0.0: version "1.0.0" @@ -1413,14 +1427,14 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== -postcss@8.4.5: - version "8.4.5" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.5.tgz#bae665764dfd4c6fcc24dc0fdf7e7aa00cc77f95" - integrity sha512-jBDboWM8qpaqwkMwItqTQTiFikhs/67OYVvblFFTM7MrZjt6yMKd6r2kgXizEbTTljacm4NldIlZnhbjr84QYg== +postcss@8.4.31: + version "8.4.31" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d" + integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== dependencies: - nanoid "^3.1.30" + nanoid "^3.3.6" picocolors "^1.0.0" - source-map-js "^1.0.1" + source-map-js "^1.0.2" prelude-ls@^1.2.1: version "1.2.1" @@ -1594,11 +1608,21 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -"source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.1: +"source-map-js@>=0.6.2 <2.0.0": version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== +source-map-js@^1.0.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.0.tgz#16b809c162517b5b8c3e7dcd315a2a5c2612b2af" + integrity sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg== + +streamsearch@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" + integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== + string.prototype.matchall@^4.0.6: version "4.0.6" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.6.tgz#5abb5dabc94c7b0ea2380f65ba610b3a544b15fa" @@ -1646,10 +1670,12 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== -styled-jsx@5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/styled-jsx/-/styled-jsx-5.0.0.tgz#816b4b92e07b1786c6b7111821750e0ba4d26e77" - integrity sha512-qUqsWoBquEdERe10EW8vLp3jT25s/ssG1/qX5gZ4wu15OZpmSMFI2v+fWlRhLfykA5rFtlJ1ME8A8pm/peV4WA== +styled-jsx@5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/styled-jsx/-/styled-jsx-5.1.1.tgz#839a1c3aaacc4e735fed0781b8619ea5d0009d1f" + integrity sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw== + dependencies: + client-only "0.0.1" supports-color@^7.1.0: version "7.2.0" @@ -1690,6 +1716,11 @@ tslib@^1.8.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== +tslib@^2.4.0: + version "2.6.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" + integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" @@ -1709,10 +1740,10 @@ type-fest@^0.20.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -typescript@5.2.2: - version "5.2.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" - integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== +typescript@5.5.2: + version "5.5.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.2.tgz#c26f023cb0054e657ce04f72583ea2d85f8d0507" + integrity sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew== unbox-primitive@^1.0.1: version "1.0.1" @@ -1731,13 +1762,6 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -use-subscription@1.5.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/use-subscription/-/use-subscription-1.5.1.tgz#73501107f02fad84c6dd57965beb0b75c68c42d1" - integrity sha512-Xv2a1P/yReAjAbhylMfFplFKj9GssgTwN7RlcTxBujFQcloStWNDQdc4g4NRWH9xS4i/FDk04vQBptAXoF3VcA== - dependencies: - object-assign "^4.1.1" - v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index ca72153e78..b95fa348f8 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -96,10 +96,13 @@ EXPOSE 5984 4369 9100 CMD ["/opt/couchdb/bin/couchdb"] FROM base as runner +ARG TARGETARCH +ENV TARGETARCH $TARGETARCH ENV COUCHDB_USER admin ENV COUCHDB_PASSWORD admin EXPOSE 5984 +EXPOSE 4984 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ @@ -125,7 +128,12 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./ WORKDIR /opt/couchdb ADD couch/vm.args couch/local.ini ./etc/ +# setup SQS +WORKDIR /opt/sqs +ADD sqs ./ +RUN chmod +x ./install.sh && ./install.sh + WORKDIR / ADD runner.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau -CMD ["./bbcouch-runner.sh"] \ No newline at end of file +RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs +CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/Dockerfile.v2 b/hosting/couchdb/Dockerfile.v2 deleted file mode 100644 index 126742cadb..0000000000 --- a/hosting/couchdb/Dockerfile.v2 +++ /dev/null @@ -1,139 +0,0 @@ -# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile -# -# Everything in this `base` image is adapted from the official `couchdb` image's -# Dockerfile. Only modifications related to upgrading from Debian bullseye to -# bookworm have been included. The `runner` image contains Budibase's -# customisations to the image, e.g. adding Clouseau. -FROM node:20-slim AS base - -# Add CouchDB user account to make sure the IDs are assigned consistently -RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb - -# be sure GPG and apt-transport-https are available and functional -RUN set -ex; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - dirmngr \ - gnupg \ - ; \ - rm -rf /var/lib/apt/lists/* - -# grab tini for signal handling and zombie reaping -# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407 -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends tini; \ - rm -rf /var/lib/apt/lists/*; \ - tini --version - -# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages -ENV GPG_COUCH_KEY \ -# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) - 390EF70BB1EA12B2773962950EE62FB37A00258D -RUN set -eux; \ - apt-get update; \ - apt-get install -y curl; \ - export GNUPGHOME="$(mktemp -d)"; \ - curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \ - gpg --batch --import keys.asc; \ - gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \ - command -v gpgconf && gpgconf --kill all || :; \ - rm -rf "$GNUPGHOME"; \ - apt-key list; \ - apt purge -y --autoremove curl; \ - rm -rf /var/lib/apt/lists/* - -ENV COUCHDB_VERSION 3.3.3 - -RUN . /etc/os-release; \ - echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \ - tee /etc/apt/sources.list.d/couchdb.list >/dev/null - -# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian -RUN set -eux; \ - apt-get update; \ - \ - echo "couchdb couchdb/mode select none" | debconf-set-selections; \ -# we DO want recommends this time - DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ - couchdb="$COUCHDB_VERSION"~bookworm \ - ; \ -# Undo symlinks to /var/log and /var/lib - rmdir /var/lib/couchdb /var/log/couchdb; \ - rm /opt/couchdb/data /opt/couchdb/var/log; \ - mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \ - chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \ - chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \ -# Remove file that sets logging to a file - rm /opt/couchdb/etc/default.d/10-filelog.ini; \ -# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh - find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \ -# Setup directories and permissions for config. Technically these could be 555 and 444 respectively -# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh. - find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \ - find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \ -# only local.d needs to be writable for the docker_entrypoint.sh - chmod -f 0777 /opt/couchdb/etc/local.d; \ -# apt clean-up - rm -rf /var/lib/apt/lists/*; - -# Add configuration -COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/ -# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ - -COPY docker-entrypoint.sh /usr/local/bin -RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat -ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"] - -VOLUME /opt/couchdb/data - -# 5984: Main CouchDB endpoint -# 4369: Erlang portmap daemon (epmd) -# 9100: CouchDB cluster communication port -EXPOSE 5984 4369 9100 -CMD ["/opt/couchdb/bin/couchdb"] - -FROM base as runner -ARG TARGETARCH -ENV TARGETARCH $TARGETARCH - -ENV COUCHDB_USER admin -ENV COUCHDB_PASSWORD admin -EXPOSE 5984 -EXPOSE 4984 - -RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ - wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ - apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \ - apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \ - apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \ - apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \ - rm -rf /var/lib/apt/lists/ - -# setup clouseau -WORKDIR / -RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \ - unzip clouseau-2.21.0-dist.zip && \ - mv clouseau-2.21.0 /opt/clouseau && \ - rm clouseau-2.21.0-dist.zip - -WORKDIR /opt/clouseau -RUN mkdir ./bin -ADD clouseau/clouseau ./bin/ -ADD clouseau/log4j.properties clouseau/clouseau.ini ./ - -# setup CouchDB -WORKDIR /opt/couchdb -ADD couch/vm.args couch/local.ini ./etc/ - -# setup SQS -WORKDIR /opt/sqs -ADD sqs ./ -RUN chmod +x ./install.sh && ./install.sh - -WORKDIR / -ADD runner.v2.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs -CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index aaadee6b43..f8cbe49b8f 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -70,9 +70,12 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & # Start CouchDB. -/docker-entrypoint.sh /opt/couchdb/bin/couchdb & +/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 & -# Wati for CouchDB to start up. +# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues. +/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 & + +# Wait for CouchDB to start up. while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do echo 'Waiting for CouchDB to start...'; sleep 5; @@ -82,4 +85,4 @@ done # function correctly, so we create them here. curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator -sleep infinity \ No newline at end of file +sleep infinity diff --git a/hosting/couchdb/runner.v2.sh b/hosting/couchdb/runner.v2.sh deleted file mode 100644 index f8cbe49b8f..0000000000 --- a/hosting/couchdb/runner.v2.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -DATA_DIR=${DATA_DIR:-/data} -COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7} - -mkdir -p ${DATA_DIR} -mkdir -p ${DATA_DIR}/couch/{dbs,views} -mkdir -p ${DATA_DIR}/search -chown -R couchdb:couchdb ${DATA_DIR}/couch - -echo ${TARGETBUILD} > /buildtarget.txt -if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persistent data & SSH on port 2222 - DATA_DIR="${DATA_DIR:-/home}" - WEBSITES_ENABLE_APP_SERVICE_STORAGE=true - mkdir -p $DATA_DIR/{search,minio,couch} - mkdir -p $DATA_DIR/couch/{dbs,views} - chown -R couchdb:couchdb $DATA_DIR/couch/ - apt update - apt-get install -y openssh-server - echo "root:Docker!" | chpasswd - mkdir -p /tmp - chmod +x /tmp/ssh_setup.sh \ - && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null) - cp /etc/sshd_config /etc/ssh/sshd_config - /etc/init.d/ssh restart - sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini -elif [[ "${TARGETBUILD}" = "single" ]]; then - # In the single image build, the Dockerfile specifies /data as a volume - # mount, so we use that for all persistent data. - sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini -elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then - # We remove the database_dir and view_index_dir settings from the local.ini - # in docker-compose because it will default to /opt/couchdb/data which is what - # our docker-compose was using prior to us switching to using our own CouchDB - # image. - sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini - sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini - sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini -elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then - # In Kubernetes the directory /opt/couchdb/data has a persistent volume - # mount for storing database data. - sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini - - # We remove the database_dir and view_index_dir settings from the local.ini - # in Kubernetes because it will default to /opt/couchdb/data which is what - # our Helm chart was using prior to us switching to using our own CouchDB - # image. - sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini - sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini - - # We remove the -name setting from the vm.args file in Kubernetes because - # it will default to the pod FQDN, which is what's required for clustering - # to work. - sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args -else - # For all other builds, we use /data for persistent data. - sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini -fi - -sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args -sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini - -# Start Clouseau. Budibase won't function correctly without Clouseau running, it -# powers the search API endpoints which are used to do all sorts, including -# populating app grids. -/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & - -# Start CouchDB. -/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 & - -# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues. -/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 & - -# Wait for CouchDB to start up. -while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do - echo 'Waiting for CouchDB to start...'; - sleep 5; -done - -# CouchDB needs the `_users` and `_replicator` databases to exist before it will -# function correctly, so we create them here. -curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users -curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator -sleep infinity diff --git a/lerna.json b/lerna.json index 32b7d8f766..efcbb7694c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,12 +1,12 @@ { - "version": "2.29.3", + "$schema": "node_modules/lerna/schemas/lerna-schema.json", + "version": "2.29.20", "npmClient": "yarn", "packages": [ "packages/*", "!packages/account-portal", "packages/account-portal/packages/*" ], - "useNx": true, "concurrency": 20, "command": { "publish": { diff --git a/nx.json b/nx.json index 8ba8798946..54db3a24a3 100644 --- a/nx.json +++ b/nx.json @@ -1,4 +1,5 @@ { + "$schema": "./node_modules/nx/schemas/nx-schema.json", "tasksRunnerOptions": { "default": { "runner": "nx-cloud", @@ -11,5 +12,10 @@ "build": { "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } + }, + "namedInputs": { + "default": ["{projectRoot}/**/*", "sharedGlobals"], + "sharedGlobals": [], + "production": ["default"] } } diff --git a/package.json b/package.json index d4a51f2e62..29b87898ac 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,8 @@ "eslint-plugin-svelte": "^2.34.0", "husky": "^8.0.3", "kill-port": "^1.6.1", - "lerna": "7.1.1", + "lerna": "7.4.2", "madge": "^6.0.0", - "nx": "16.4.3", "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", @@ -34,10 +33,10 @@ "scripts": { "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", - "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", - "build:apps": "yarn build --scope @budibase/server --scope @budibase/worker", + "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", + "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker", + "build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", "build:cli": "yarn build --scope @budibase/cli", - "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server", @@ -78,7 +77,6 @@ "build:docker:single:sqs": "./scripts/build-single-image-sqs.sh", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 --push ./hosting/couchdb", - "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.3.3-sqs --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run --stream env:multi:enable", diff --git a/packages/account-portal b/packages/account-portal index ff16525b73..b03e584e46 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ff16525b73c5751d344f5c161a682609c0a993f2 +Subproject commit b03e584e465f620b49a1b688ff4afc973e6c0758 diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 88b970884c..bf5215a724 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -22,10 +22,9 @@ }, "dependencies": { "@budibase/nano": "10.1.5", - "@budibase/pouchdb-replication-stream": "1.2.10", + "@budibase/pouchdb-replication-stream": "1.2.11", "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", - "@govtechsg/passport-openidconnect": "^1.0.2", "aws-cloudfront-sign": "3.0.2", "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts deleted file mode 100644 index 69c98fe569..0000000000 --- a/packages/backend-core/src/db/constants.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { - CONSTANT_INTERNAL_ROW_COLS, - CONSTANT_EXTERNAL_ROW_COLS, - isInternalColumnName, -} from "@budibase/shared-core" diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 4db63ad695..f3c3beeaab 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -13,6 +13,7 @@ import { isDocument, RowResponse, RowValue, + SqlClient, SQLiteDefinition, SqlQueryBinding, } from "@budibase/types" @@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import env from "../../environment" +import { sqlLog } from "../../sql/utils" const DATABASE_NOT_FOUND = "Database does not exist." @@ -80,6 +82,11 @@ export function DatabaseWithConnection( connection: string, opts?: DatabaseOpts ) { + if (!dbName || !connection) { + throw new Error( + "Unable to create database without database name or connection" + ) + } const db = new DatabaseImpl(dbName, opts, connection) return new DDInstrumentedDatabase(db) } @@ -317,6 +324,7 @@ export class DatabaseImpl implements Database { ): Promise { const dbName = this.name const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}` + sqlLog(SqlClient.SQL_LITE, sql, parameters) return await this._sqlQuery(url, "POST", { query: sql, args: parameters, diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts index 8dbbe34e3a..5c7d7ec81d 100644 --- a/packages/backend-core/src/db/couch/connections.ts +++ b/packages/backend-core/src/db/couch/connections.ts @@ -1,6 +1,7 @@ import env from "../../environment" export const getCouchInfo = (connection?: string) => { + // clean out any auth credentials const urlInfo = getUrlInfo(connection) let username let password @@ -23,9 +24,19 @@ export const getCouchInfo = (connection?: string) => { throw new Error("CouchDB password not set") } const authCookie = Buffer.from(`${username}:${password}`).toString("base64") + let sqlUrl = env.COUCH_DB_SQL_URL + // default for dev + if (env.isDev() && !sqlUrl) { + sqlUrl = "http://localhost:4006" + } else if (!sqlUrl && urlInfo.url) { + const parsed = new URL(urlInfo.url) + // attempt to connect on default port + sqlUrl = urlInfo.url.replace(parsed.port, "4984") + } return { url: urlInfo.url!, - sqlUrl: env.COUCH_DB_SQL_URL, + // clean out any auth credentials + sqlUrl: getUrlInfo(sqlUrl).url, auth: { username: username, password: password, diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 932efed3f7..c731d20d6c 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,4 +2,3 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" -export * from "../constants" diff --git a/packages/backend-core/src/db/tests/connections.spec.ts b/packages/backend-core/src/db/tests/connections.spec.ts new file mode 100644 index 0000000000..2130467292 --- /dev/null +++ b/packages/backend-core/src/db/tests/connections.spec.ts @@ -0,0 +1,22 @@ +import env from "../../environment" +import { getCouchInfo } from "../couch" + +const MAIN_COUCH_URL = "http://user:test@localhost:5984" + +describe("connections", () => { + beforeAll(() => { + env._set("COUCH_DB_SQL_URL", "https://user:test@localhost:4984") + }) + + it("should strip URL credentials", () => { + const response = getCouchInfo(MAIN_COUCH_URL) + expect(response.url).toBe("http://localhost:5984") + expect(response.sqlUrl).toBe("https://localhost:4984") + }) + + it("should return separate auth credentials", () => { + const response = getCouchInfo(MAIN_COUCH_URL) + expect(response.auth.username).toBe("user") + expect(response.auth.password).toBe("test") + }) +}) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 906a95e1db..a7ad453066 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,6 +1,6 @@ import env from "../environment" import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants" -import { getTenantId, getGlobalDBName } from "../context" +import { getTenantId, getGlobalDBName, isMultiTenant } from "../context" import { doWithDB, directCouchAllDbs } from "./db" import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions" @@ -206,3 +206,34 @@ export function pagination( nextPage, } } + +export function isSqsEnabledForTenant(): boolean { + const tenantId = getTenantId() + if (!env.SQS_SEARCH_ENABLE) { + return false + } + + // single tenant (self host and dev) always enabled if flag set + if (!isMultiTenant()) { + return true + } + + // This is to guard against the situation in tests where tests pass because + // we're not actually using SQS, we're using Lucene and the tests pass due to + // parity. + if (env.isTest() && env.SQS_SEARCH_ENABLE_TENANTS.length === 0) { + throw new Error( + "to enable SQS you must specify a list of tenants in the SQS_SEARCH_ENABLE_TENANTS env var" + ) + } + + // Special case to enable all tenants, for testing in QA. + if ( + env.SQS_SEARCH_ENABLE_TENANTS.length === 1 && + env.SQS_SEARCH_ENABLE_TENANTS[0] === "*" + ) { + return true + } + + return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId) +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index e58660a889..64e3187956 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -114,8 +114,11 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", - COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL, SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, + SQS_SEARCH_ENABLE_TENANTS: + process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [], + SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, @@ -200,8 +203,28 @@ const environment = { }, ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M", DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS, + BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, + BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, } +type EnvironmentKey = keyof typeof environment +export const SECRETS: EnvironmentKey[] = [ + "API_ENCRYPTION_KEY", + "BB_ADMIN_USER_PASSWORD", + "COUCH_DB_PASSWORD", + "COUCH_DB_SQL_URL", + "COUCH_DB_URL", + "GOOGLE_CLIENT_SECRET", + "INTERNAL_API_KEY_FALLBACK", + "INTERNAL_API_KEY", + "JWT_SECRET", + "MINIO_ACCESS_KEY", + "MINIO_SECRET_KEY", + "OPENAI_API_KEY", + "REDIS_PASSWORD", +] + // clean up any environment variable edge cases for (let [key, value] of Object.entries(environment)) { // handle the edge case of "0" to disable an environment variable diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 08f9f3214d..6ceda9cd3a 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -1,6 +1,7 @@ import { APIError } from "@budibase/types" import * as errors from "../errors" import environment from "../environment" +import { stringContainsSecret } from "../security/secrets" export async function errorHandling(ctx: any, next: any) { try { @@ -17,11 +18,19 @@ export async function errorHandling(ctx: any, next: any) { let error: APIError = { message: err.message, - status: status, + status, validationErrors: err.validation, error: errors.getPublicError(err), } + if (stringContainsSecret(JSON.stringify(error))) { + error = { + message: "Unexpected error", + status, + error: "Unexpected error", + } + } + if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) { // @ts-ignore error.stack = err.stack diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 4d11caf220..9160c6a6dd 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -2,6 +2,7 @@ import { GenericContainer, StartedTestContainer } from "testcontainers" import { generator, structures } from "../../../tests" import RedisWrapper from "../redis" import { env } from "../.." +import { randomUUID } from "crypto" jest.setTimeout(30000) @@ -52,10 +53,10 @@ describe("redis", () => { describe("bulkStore", () => { function createRandomObject( keyLength: number, - valueGenerator: () => any = () => generator.word() + valueGenerator: () => any = () => randomUUID() ) { return generator - .unique(() => generator.word(), keyLength) + .unique(() => randomUUID(), keyLength) .reduce((acc, key) => { acc[key] = valueGenerator() return acc diff --git a/packages/backend-core/src/security/secrets.ts b/packages/backend-core/src/security/secrets.ts new file mode 100644 index 0000000000..65bc33a1dc --- /dev/null +++ b/packages/backend-core/src/security/secrets.ts @@ -0,0 +1,20 @@ +import environment, { SECRETS } from "../environment" + +export function stringContainsSecret(str: string) { + if (str.includes("-----BEGIN PRIVATE KEY-----")) { + return true + } + + for (const key of SECRETS) { + const value = environment[key] + if (typeof value !== "string" || value === "") { + continue + } + + if (str.includes(value)) { + return true + } + } + + return false +} diff --git a/packages/backend-core/src/security/tests/secrets.spec.ts b/packages/backend-core/src/security/tests/secrets.spec.ts new file mode 100644 index 0000000000..19bf174973 --- /dev/null +++ b/packages/backend-core/src/security/tests/secrets.spec.ts @@ -0,0 +1,35 @@ +import { randomUUID } from "crypto" +import environment, { SECRETS } from "../../environment" +import { stringContainsSecret } from "../secrets" + +describe("secrets", () => { + describe("stringContainsSecret", () => { + it.each(SECRETS)("detects that a string contains a secret in: %s", key => { + const needle = randomUUID() + const haystack = `this is a secret: ${needle}` + const old = environment[key] + environment._set(key, needle) + + try { + expect(stringContainsSecret(haystack)).toBe(true) + } finally { + environment._set(key, old) + } + }) + + it.each(SECRETS)( + "detects that a string does not contain a secret in: %s", + key => { + const needle = randomUUID() + const haystack = `this does not contain a secret` + const old = environment[key] + environment._set(key, needle) + try { + expect(stringContainsSecret(haystack)).toBe(false) + } finally { + environment._set(key, old) + } + } + ) + }) +}) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 34b950bf2c..4936e4da68 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,16 +3,20 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isIsoDateString, + isValidISODateString, isValidFilter, + sqlLog, + isInvalidISODateString, } from "./utils" import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" import { + AnySearchFilter, BBReferenceFieldMetadata, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, + InternalSearchFilterOperator, JsonFieldMetadata, JsonTypes, Operation, @@ -38,11 +42,7 @@ const envLimit = environment.SQL_MAX_ROWS : null const BASE_LIMIT = envLimit || 5000 -// these are invalid dates sent by the client, need to convert them to a real max date -const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z" -const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z" - -function likeKey(client: string, key: string): string { +function likeKey(client: string | string[], key: string): string { let start: string, end: string switch (client) { case SqlClient.MY_SQL: @@ -75,10 +75,10 @@ function parse(input: any) { if (typeof input !== "string") { return input } - if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) { + if (isInvalidISODateString(input)) { return null } - if (isIsoDateString(input)) { + if (isValidISODateString(input)) { return new Date(input.trim()) } return input @@ -184,7 +184,11 @@ class InternalBuilder { query: Knex.QueryBuilder, filters: SearchFilters | undefined, table: Table, - opts: { aliases?: Record; relationship?: boolean } + opts: { + aliases?: Record + relationship?: boolean + columnPrefix?: string + } ): Knex.QueryBuilder { if (!filters) { return query @@ -192,7 +196,10 @@ class InternalBuilder { filters = parseFilters(filters) // if all or specified in filters, then everything is an or const allOr = filters.allOr - const sqlStatements = new SqlStatements(this.client, table, { allOr }) + const sqlStatements = new SqlStatements(this.client, table, { + allOr, + columnPrefix: opts.columnPrefix, + }) const tableName = this.client === SqlClient.SQL_LITE ? table._id! : table.name @@ -201,17 +208,32 @@ class InternalBuilder { return alias || name } function iterate( - structure: { [key: string]: any }, - fn: (key: string, value: any) => void + structure: AnySearchFilter, + fn: (key: string, value: any) => void, + complexKeyFn?: (key: string[], value: any) => void ) { - for (let [key, value] of Object.entries(structure)) { + for (const key in structure) { + const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") - if (!opts.relationship && !isRelationshipField) { + + let castedTypeValue + if ( + key === InternalSearchFilterOperator.COMPLEX_ID_OPERATOR && + (castedTypeValue = structure[key]) && + complexKeyFn + ) { + const alias = getTableAlias(tableName) + complexKeyFn( + castedTypeValue.id.map((x: string) => + alias ? `${alias}.${x}` : x + ), + castedTypeValue.values + ) + } else if (!opts.relationship && !isRelationshipField) { const alias = getTableAlias(tableName) fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) - } - if (opts.relationship && isRelationshipField) { + } else if (opts.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") const alias = getTableAlias(filterTableName) fn(alias ? `${alias}.${property}` : property, value) @@ -234,7 +256,7 @@ class InternalBuilder { } } - const contains = (mode: object, any: boolean = false) => { + const contains = (mode: AnySearchFilter, any: boolean = false) => { const rawFnc = allOr ? "orWhereRaw" : "whereRaw" const not = mode === filters?.notContains ? "NOT " : "" function stringifyArray(value: Array, quoteStyle = '"'): string { @@ -246,7 +268,7 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (key: string, value: Array) => { + iterate(mode, (key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) @@ -261,7 +283,7 @@ class InternalBuilder { }) } else if (this.client === SqlClient.MY_SQL) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (key: string, value: Array) => { + iterate(mode, (key, value) => { query = query[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value @@ -270,7 +292,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (key: string, value: Array) => { + iterate(mode, (key, value) => { let statement = "" for (let i in value) { if (typeof value[i] === "string") { @@ -294,10 +316,16 @@ class InternalBuilder { } if (filters.oneOf) { - iterate(filters.oneOf, (key, array) => { - const fnc = allOr ? "orWhereIn" : "whereIn" - query = query[fnc](key, Array.isArray(array) ? array : [array]) - }) + const fnc = allOr ? "orWhereIn" : "whereIn" + iterate( + filters.oneOf, + (key: string, array) => { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + }, + (key: string[], array) => { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + } + ) } if (filters.string) { iterate(filters.string, (key, value) => { @@ -663,6 +691,7 @@ class InternalBuilder { } // add filters to the query (where) query = this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, aliases: tableAliases, }) @@ -698,6 +727,7 @@ class InternalBuilder { } return this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, relationship: true, aliases: tableAliases, }) @@ -708,6 +738,7 @@ class InternalBuilder { let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) query = this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, aliases: tableAliases, }) // mysql can't use returning @@ -722,6 +753,7 @@ class InternalBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) query = this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, aliases: tableAliases, }) // mysql can't use returning @@ -735,6 +767,7 @@ class InternalBuilder { class SqlQueryBuilder extends SqlTableQueryBuilder { private readonly limit: number + // pass through client to get flavour of SQL constructor(client: string, limit: number = BASE_LIMIT) { super(client) @@ -927,15 +960,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { } log(query: string, values?: SqlQueryBinding) { - if (!environment.SQL_LOGGING_ENABLE) { - return - } - const sqlClient = this.getSqlClient() - let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"` - if (values) { - string += ` values="${values.join(", ")}"` - } - console.log(string) + sqlLog(this.getSqlClient(), query, values) } } diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts index a80defd8b8..311f7c7d49 100644 --- a/packages/backend-core/src/sql/sqlStatements.ts +++ b/packages/backend-core/src/sql/sqlStatements.ts @@ -5,19 +5,27 @@ export class SqlStatements { client: string table: Table allOr: boolean | undefined + columnPrefix: string | undefined + constructor( client: string, table: Table, - { allOr }: { allOr?: boolean } = {} + { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {} ) { this.client = client this.table = table this.allOr = allOr + this.columnPrefix = columnPrefix } getField(key: string): FieldSchema | undefined { const fieldName = key.split(".")[1] - return this.table.schema[fieldName] + let found = this.table.schema[fieldName] + if (!found && this.columnPrefix) { + const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "") + found = this.table.schema[prefixRemovedFieldName] + } + return found } between( diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 45ab510948..e73c6ac445 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" import { DEFAULT_BB_DATASOURCE_ID } from "../constants" import { Knex } from "knex" import { SEPARATOR } from "../db" +import environment from "../environment" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g const ENCODED_SPACE = encodeURIComponent(" ") +const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/ export function isExternalTableID(tableId: string) { return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR) @@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } } -export function isIsoDateString(str: string) { +export function isInvalidISODateString(str: string) { const trimmedValue = str.trim() - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { + if (!ISO_DATE_REGEX.test(trimmedValue)) { return false } let d = new Date(trimmedValue) + return isNaN(d.getTime()) +} + +export function isValidISODateString(str: string) { + const trimmedValue = str.trim() + if (!ISO_DATE_REGEX.test(trimmedValue)) { + return false + } + let d = new Date(trimmedValue) + if (isNaN(d.getTime())) { + return false + } return d.toISOString() === trimmedValue } export function isValidFilter(value: any) { return value != null && value !== "" } + +export function sqlLog(client: string, query: string, values?: any[]) { + if (!environment.SQL_LOGGING_ENABLE) { + return + } + let string = `[SQL] [${client.toUpperCase()}] query="${query}"` + if (values) { + string += ` values="${values.join(", ")}"` + } + console.log(string) +} diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 37547573bd..4865ebb5bc 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -221,7 +221,7 @@ export class UserDB { const tenantId = getTenantId() const db = getGlobalDB() - let { email, _id, userGroups = [], roles } = user + const { email, _id, userGroups = [], roles } = user if (!email && !_id) { throw new Error("_id or email is required") @@ -231,11 +231,10 @@ export class UserDB { if (_id) { // try to get existing user from db try { - dbUser = (await db.get(_id)) as User - if (email && dbUser.email !== email) { - throw "Email address cannot be changed" + dbUser = await usersCore.getById(_id) + if (email && dbUser.email !== email && !opts.allowChangingEmail) { + throw new Error("Email address cannot be changed") } - email = dbUser.email } catch (e: any) { if (e.status === 404) { // do nothing, save this new user with the id specified - required for SSO auth @@ -271,13 +270,13 @@ export class UserDB { // make sure we set the _id field for a new user // Also if this is a new user, associate groups with them - let groupPromises = [] + const groupPromises = [] if (!_id) { - _id = builtUser._id! - if (userGroups.length > 0) { for (let groupId of userGroups) { - groupPromises.push(UserDB.groups.addUsers(groupId, [_id!])) + groupPromises.push( + UserDB.groups.addUsers(groupId, [builtUser._id!]) + ) } } } @@ -288,6 +287,11 @@ export class UserDB { builtUser._rev = response.rev await eventHelpers.handleSaveEvents(builtUser, dbUser) + if (dbUser && builtUser.email !== dbUser.email) { + // Remove the plaform email reference if the email changed + await platform.users.removeUser({ email: dbUser.email } as User) + } + await platform.users.addUser( tenantId, builtUser._id!, diff --git a/packages/backend-core/src/users/test/db.spec.ts b/packages/backend-core/src/users/test/db.spec.ts new file mode 100644 index 0000000000..3e29d6673c --- /dev/null +++ b/packages/backend-core/src/users/test/db.spec.ts @@ -0,0 +1,188 @@ +import { User, UserStatus } from "@budibase/types" +import { DBTestConfiguration, generator, structures } from "../../../tests" +import { UserDB } from "../db" +import { searchExistingEmails } from "../lookup" + +const db = UserDB + +const config = new DBTestConfiguration() + +const quotas = { + addUsers: jest + .fn() + .mockImplementation( + (_change: number, _creatorsChange: number, cb?: () => Promise) => + cb && cb() + ), + removeUsers: jest + .fn() + .mockImplementation( + (_change: number, _creatorsChange: number, cb?: () => Promise) => + cb && cb() + ), +} +const groups = { + addUsers: jest.fn(), + getBulk: jest.fn(), + getGroupBuilderAppIds: jest.fn(), +} +const features = { isSSOEnforced: jest.fn(), isAppBuildersEnabled: jest.fn() } + +describe("UserDB", () => { + beforeAll(() => { + db.init(quotas, groups, features) + }) + + describe("save", () => { + describe("create", () => { + it("creating a new user will persist it", async () => { + const email = generator.email({}) + const user: User = structures.users.user({ + email, + tenantId: config.getTenantId(), + }) + + await config.doInTenant(async () => { + const saveUserResponse = await db.save(user) + + const persistedUser = await db.getUserByEmail(email) + expect(persistedUser).toEqual({ + ...user, + _id: saveUserResponse._id, + _rev: expect.stringMatching(/^1-\w+/), + password: expect.not.stringMatching(user.password!), + status: UserStatus.ACTIVE, + createdAt: Date.now(), + updatedAt: new Date().toISOString(), + }) + }) + }) + + it("the same email cannot be used twice in the same tenant", async () => { + const email = generator.email({}) + const user: User = structures.users.user({ + email, + tenantId: config.getTenantId(), + }) + + await config.doInTenant(() => db.save(user)) + + await config.doInTenant(() => + expect(db.save(user)).rejects.toThrow( + `Email already in use: '${email}'` + ) + ) + }) + + it("the same email cannot be used twice in different tenants", async () => { + const email = generator.email({}) + const user: User = structures.users.user({ + email, + tenantId: config.getTenantId(), + }) + + await config.doInTenant(() => db.save(user)) + + config.newTenant() + await config.doInTenant(() => + expect(db.save(user)).rejects.toThrow( + `Email already in use: '${email}'` + ) + ) + }) + }) + + describe("update", () => { + let user: User + + beforeEach(async () => { + user = await config.doInTenant(() => + db.save( + structures.users.user({ + email: generator.email({}), + tenantId: config.getTenantId(), + }) + ) + ) + }) + + it("can update user properties", async () => { + await config.doInTenant(async () => { + const updatedName = generator.first() + user.firstName = updatedName + + await db.save(user) + + const persistedUser = await db.getUserByEmail(user.email) + expect(persistedUser).toEqual( + expect.objectContaining({ + _id: user._id, + email: user.email, + firstName: updatedName, + lastName: user.lastName, + }) + ) + }) + }) + + it("email cannot be updated by default", async () => { + await config.doInTenant(async () => { + await expect( + db.save({ ...user, email: generator.email({}) }) + ).rejects.toThrow("Email address cannot be changed") + }) + }) + + it("email can be updated if specified", async () => { + await config.doInTenant(async () => { + const newEmail = generator.email({}) + + await db.save( + { ...user, email: newEmail }, + { allowChangingEmail: true } + ) + + const persistedUser = await db.getUserByEmail(newEmail) + expect(persistedUser).toEqual( + expect.objectContaining({ + _id: user._id, + email: newEmail, + lastName: user.lastName, + _rev: expect.stringMatching(/^2-\w+/), + }) + ) + }) + }) + + it("updating emails frees previous emails", async () => { + await config.doInTenant(async () => { + const previousEmail = user.email + const newEmail = generator.email({}) + expect(await searchExistingEmails([previousEmail, newEmail])).toEqual( + [previousEmail] + ) + + await db.save( + { ...user, email: newEmail }, + { allowChangingEmail: true } + ) + + expect(await searchExistingEmails([previousEmail, newEmail])).toEqual( + [newEmail] + ) + + await db.save( + structures.users.user({ + email: previousEmail, + tenantId: config.getTenantId(), + }) + ) + + expect(await searchExistingEmails([previousEmail, newEmail])).toEqual( + [previousEmail, newEmail] + ) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 7d62a6ef39..0c994d8287 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -18,9 +18,10 @@ import { CouchFindOptions, DatabaseQueryOpts, SearchFilters, - SearchFilterOperator, SearchUsersRequest, User, + BasicOperator, + ArrayOperator, } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" @@ -46,9 +47,9 @@ function removeUserPassword(users: User | User[]) { export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ - { op: SearchFilterOperator.STRING, key: "email" }, - { op: SearchFilterOperator.EQUAL, key: "_id" }, - { op: SearchFilterOperator.ONE_OF, key: "_id" }, + { op: BasicOperator.STRING, key: "email" }, + { op: BasicOperator.EQUAL, key: "_id" }, + { op: ArrayOperator.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index 4a3da8db8c..a49c2a795e 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,4 +1,7 @@ -import { db } from "../../../src" +import { + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" export function expectFunctionWasCalledTimesWith( jestFunction: any, @@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith( } export const expectAnyInternalColsAttributes: { - [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), type: expect.anything(), @@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: { } export const expectAnyExternalColsAttributes: { - [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), _id: expect.anything(), diff --git a/packages/bbui/src/Drawer/Drawer.svelte b/packages/bbui/src/Drawer/Drawer.svelte index 89ee92726d..1f38389a63 100644 --- a/packages/bbui/src/Drawer/Drawer.svelte +++ b/packages/bbui/src/Drawer/Drawer.svelte @@ -223,7 +223,7 @@ height: 420px; background: var(--background); border: var(--border-light); - z-index: 100; + z-index: 1000; border-radius: 8px; overflow: hidden; box-sizing: border-box; diff --git a/packages/bbui/src/Tooltip/TooltipWrapper.svelte b/packages/bbui/src/Tooltip/TooltipWrapper.svelte index 4c20cb54a0..8d12b88086 100644 --- a/packages/bbui/src/Tooltip/TooltipWrapper.svelte +++ b/packages/bbui/src/Tooltip/TooltipWrapper.svelte @@ -1,33 +1,25 @@ -
{#if tooltip}
-
(showTooltip = true)} - on:mouseleave={() => (showTooltip = false)} - on:focus - > - -
- {#if showTooltip} -
- + +
+
- {/if} +
{/if}
@@ -44,14 +36,6 @@ margin-left: 5px; margin-right: 5px; } - .tooltip { - position: absolute; - display: flex; - justify-content: center; - top: 15px; - z-index: 200; - width: 160px; - } .icon { transform: scale(0.75); } diff --git a/packages/builder/package.json b/packages/builder/package.json index a00936bdca..f44c2ea549 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -74,7 +74,7 @@ "lodash": "4.17.21", "posthog-js": "^1.118.0", "remixicon": "2.5.0", - "sanitize-html": "^2.7.0", + "sanitize-html": "^2.13.0", "shortid": "2.2.15", "svelte-dnd-action": "^0.9.8", "svelte-loading-spinners": "^0.1.1", diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte index d68e57ca36..f79b36b1ca 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte @@ -112,7 +112,7 @@ This action cannot be undone. - + @@ -148,7 +148,6 @@ .header.scrolling { background: var(--background); border-bottom: var(--border-light); - border-left: var(--border-light); z-index: 1; } diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte index d212300cdf..7d223299c7 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte @@ -8,11 +8,63 @@ import { automationStore, selectedAutomation } from "stores/builder" import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte" import { cloneDeep } from "lodash/fp" + import { memo } from "@budibase/frontend-core" + import { AutomationEventType } from "@budibase/types" let failedParse = null let trigger = {} let schemaProperties = {} + const rowTriggers = [ + AutomationEventType.ROW_DELETE, + AutomationEventType.ROW_UPDATE, + AutomationEventType.ROW_SAVE, + ] + + /** + * Parses the automation test data and ensures it is valid + * @param {object} testData contains all config for the test + * @returns {object} valid testData + * @todo Parse *all* data for each trigger type and relay adequate feedback + */ + const parseTestData = testData => { + const autoTrigger = $selectedAutomation?.definition?.trigger + const { tableId } = autoTrigger?.inputs || {} + + // Ensure the tableId matches the trigger table for row trigger automations + if ( + rowTriggers.includes(autoTrigger?.event) && + testData?.row?.tableId !== tableId + ) { + return { + // Reset Core fields + row: { tableId }, + meta: {}, + id: "", + revision: "", + } + } else { + // Leave the core data as it is + return testData + } + } + + /** + * Before executing a test run, relay if an automation is in a valid state + * @param {object} trigger The automation trigger config + * @returns {boolean} validation status + * @todo Parse *all* trigger types relay adequate feedback + */ + const isTriggerValid = trigger => { + if (rowTriggers.includes(trigger?.event) && !trigger?.inputs?.tableId) { + return false + } + return true + } + + const memoTestData = memo(parseTestData($selectedAutomation.testData)) + $: memoTestData.set(parseTestData($selectedAutomation.testData)) + $: { // clone the trigger so we're not mutating the reference trigger = cloneDeep($selectedAutomation.definition.trigger) @@ -20,34 +72,45 @@ // get the outputs so we can define the fields let schema = Object.entries(trigger.schema?.outputs?.properties || {}) - if (trigger?.event === "app:trigger") { + if (trigger?.event === AutomationEventType.APP_TRIGGER) { schema = [["fields", { customType: "fields" }]] } - schemaProperties = schema } - // check to see if there is existing test data in the store - $: testData = $selectedAutomation.testData || {} - // Check the schema to see if required fields have been entered - $: isError = !trigger.schema.outputs.required.every( - required => testData[required] || required !== "row" - ) + $: isError = + !isTriggerValid(trigger) || + !trigger.schema.outputs.required.every( + required => $memoTestData?.[required] || required !== "row" + ) function parseTestJSON(e) { + let jsonUpdate + try { - const obj = JSON.parse(e.detail) + jsonUpdate = JSON.parse(e.detail) failedParse = null - automationStore.actions.addTestDataToAutomation(obj) } catch (e) { failedParse = "Invalid JSON" + return false } + + if (rowTriggers.includes(trigger?.event)) { + const tableId = trigger?.inputs?.tableId + + // Reset the tableId as it must match the trigger + if (jsonUpdate?.row?.tableId !== tableId) { + jsonUpdate.row.tableId = tableId + } + } + + automationStore.actions.addTestDataToAutomation(jsonUpdate) } const testAutomation = async () => { try { - await automationStore.actions.test($selectedAutomation, testData) + await automationStore.actions.test($selectedAutomation, $memoTestData) $automationStore.showTestPanel = true } catch (error) { notifications.error(error) @@ -85,7 +148,7 @@ {#if selectedValues}
lowerB ? 1 : -1 }) + $: groupedAutomations = filteredAutomations.reduce((acc, auto) => { + acc[auto.definition.trigger.event] ??= { + icon: auto.definition.trigger.icon, + name: (auto.definition.trigger?.name || "").toUpperCase(), + entries: [], + } + acc[auto.definition.trigger.event].entries.push(auto) + return acc + }, {}) + $: showNoResults = searchString && !filteredAutomations.length onMount(async () => { @@ -55,16 +65,25 @@ />
- {#each filteredAutomations as automation} - selectAutomation(automation._id)} - selectedBy={$userSelectedResourceMap[automation._id]} - disabled={automation.disabled} - > - - + {#each Object.values(groupedAutomations || {}) as triggerGroup} +
+
+ {triggerGroup?.name} +
+ {#each triggerGroup.entries as automation} + selectAutomation(automation._id)} + selectedBy={$userSelectedResourceMap[automation._id]} + disabled={automation.disabled} + > + + + {/each} +
{/each} {#if showNoResults} @@ -82,6 +101,17 @@ diff --git a/packages/builder/src/components/automation/SetupPanel/PropField.svelte b/packages/builder/src/components/automation/SetupPanel/PropField.svelte new file mode 100644 index 0000000000..ba3c0ff081 --- /dev/null +++ b/packages/builder/src/components/automation/SetupPanel/PropField.svelte @@ -0,0 +1,60 @@ + + +
+
+ +
+
+ +
+
+ + diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index bd3bcda774..32e38803b3 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -1,28 +1,43 @@ -
- -
- onChange(e, field)} - value={value[field]} + on:change={e => + onChange({ + row: { + [field]: e.detail, + }, + })} + value={fieldData} options={schema.constraints.inclusion} /> {:else if schema.type === "datetime"} - onChange(e, field)} /> + + onChange({ + row: { + [field]: e.detail, + }, + })} + /> {:else if schema.type === "boolean"}