diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 757a9a4983..a15e3d9110 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -44,9 +44,9 @@ jobs: node-version: 14.x cache: "yarn" - run: yarn - - run: yarn build + - run: yarn nx run-many -t=build --configuration=production - test: + test-libraries: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -59,8 +59,27 @@ jobs: node-version: 14.x cache: "yarn" - run: yarn - - run: yarn build --scope=@budibase/types --scope=@budibase/shared-core --scope=@budibase/string-templates - - run: yarn test --ignore=@budibase/pro + - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro + - uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + name: codecov-umbrella + verbose: true + + test-services: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: Use Node.js 14.x + uses: actions/setup-node@v3 + with: + node-version: 14.x + cache: "yarn" + - run: yarn + - run: yarn test --scope=@budibase/worker --scope=@budibase/server - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 9b84b70de0..c724b717e2 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -37,7 +37,7 @@ jobs: with: node-version: 14.x - - run: yarn + - run: yarn install --frozen-lockfile - name: Update versions run: | version=$(cat lerna.json \ @@ -51,7 +51,7 @@ jobs: node scripts/syncLocalDependencies.js $version echo "Syncing yarn workspace" yarn - - run: yarn build + - run: yarn build --configuration=production - run: yarn build:sdk - name: Publish budibase packages to NPM diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index fdea9f76c2..4959194064 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -42,7 +42,7 @@ jobs: with: node-version: 14.x - - run: yarn + - run: yarn install --frozen-lockfile - name: Update versions run: | version=$(cat lerna.json \ @@ -57,7 +57,7 @@ jobs: echo "Syncing yarn workspace" yarn - run: yarn lint - - run: yarn build + - run: yarn build --configuration=production - run: yarn build:sdk - name: Publish budibase packages to NPM diff --git a/.nvmrc b/.nvmrc index 436b34d932..835d07c442 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v14.20.1 \ No newline at end of file +v14.20.1 diff --git a/.python-version b/.python-version index e06d07afe1..30291cba22 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.0 \ No newline at end of file +3.10.0 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6e667d23a8..ac35929be1 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -144,8 +144,6 @@ The following commands can be executed to manually get Budibase up and running ( `yarn` to install project dependencies -`yarn bootstrap` will install all budibase modules and symlink them together using lerna. - `yarn build` will build all budibase packages. #### 4. Running @@ -243,7 +241,7 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README. Note that only budibase maintainers will be able to access the pro repo. -The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. +By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. ### Troubleshooting diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml new file mode 100644 index 0000000000..391d263098 --- /dev/null +++ b/hosting/docker-compose.build.yaml @@ -0,0 +1,77 @@ +version: "3" + +# optional ports are specified throughout for more advanced use cases. + +services: + app-service: + build: ../packages/server + container_name: build-bbapps + environment: + SELF_HOSTED: 1 + COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 + WORKER_URL: http://worker-service:4003 + MINIO_URL: http://minio-service:9000 + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + INTERNAL_API_KEY: ${INTERNAL_API_KEY} + BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} + PORT: 4002 + API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} + JWT_SECRET: ${JWT_SECRET} + LOG_LEVEL: info + SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 + ENABLE_ANALYTICS: "true" + REDIS_URL: redis-service:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} + BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} + PLUGINS_DIR: ${PLUGINS_DIR} + depends_on: + - worker-service + - redis-service + # volumes: + # - /some/path/to/plugins:/plugins + + worker-service: + build: ../packages/worker + container_name: build-bbworker + environment: + SELF_HOSTED: 1 + PORT: 4003 + CLUSTER_PORT: ${MAIN_PORT} + API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} + JWT_SECRET: ${JWT_SECRET} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_URL: http://minio-service:9000 + APPS_URL: http://app-service:4002 + COUCH_DB_USERNAME: ${COUCH_DB_USER} + COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} + COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 + SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 + INTERNAL_API_KEY: ${INTERNAL_API_KEY} + REDIS_URL: redis-service:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + depends_on: + - redis-service + - minio-service + + proxy-service-docker: + ports: + - "${MAIN_PORT}:10000" + container_name: build-bbproxy + image: budibase/proxy + environment: + - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10 + - PROXY_RATE_LIMIT_API_PER_SECOND=20 + - APPS_UPSTREAM_URL=http://app-service:4002 + - WORKER_UPSTREAM_URL=http://worker-service:4003 + - MINIO_UPSTREAM_URL=http://minio-service:9000 + - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984 + - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080 + - RESOLVER=127.0.0.11 + depends_on: + - minio-service + - worker-service + - app-service + - couchdb-service diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index fadcf235e9..64a6b01365 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,22 +1,22 @@ -FROM node:14-slim as build +FROM node:16-slim as build # install node-gyp dependencies RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python # add pin script WORKDIR / -ADD scripts/pinVersions.js scripts/cleanup.sh ./ +ADD scripts/cleanup.sh ./ RUN chmod +x /cleanup.sh # build server WORKDIR /app ADD packages/server . -RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh +RUN yarn install --frozen-lockfile --production=true && /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . -RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh +RUN yarn install --frozen-lockfile --production=true && /cleanup.sh FROM budibase/couchdb ARG TARGETARCH @@ -31,9 +31,7 @@ COPY --from=build /worker /worker # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \ - apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \ - apt-get update + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs diff --git a/lerna.json b/lerna.json index 173e3b7a15..ebaf23cfb6 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.4", + "version": "2.6.19-alpha.11", "npmClient": "yarn", "packages": [ "packages/backend-core", diff --git a/nx.json b/nx.json index efb577a01b..fc0712eed4 100644 --- a/nx.json +++ b/nx.json @@ -6,5 +6,15 @@ "cacheableOperations": ["build", "test"] } } + }, + "targetDefaults": { + "dev:builder": { + "dependsOn": [ + { + "projects": ["@budibase/string-templates"], + "target": "build" + } + ] + } } } diff --git a/package.json b/package.json index 27f94ada0d..edbb82b892 100644 --- a/package.json +++ b/package.json @@ -2,17 +2,24 @@ "name": "root", "private": true, "devDependencies": { + "@esbuild-plugins/node-resolve": "^0.2.2", + "@esbuild-plugins/tsconfig-paths": "^0.1.2", + "@nx/esbuild": "16.2.1", + "@nx/js": "16.2.1", "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "5.45.0", "babel-eslint": "^10.0.3", + "esbuild": "^0.17.18", "eslint": "^7.28.0", "eslint-plugin-cypress": "^2.11.3", "eslint-plugin-svelte3": "^3.2.0", "husky": "^8.0.3", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", - "lerna": "^6.6.1", + "lerna": "7.0.0-alpha.0", "madge": "^6.0.0", + "minimist": "^1.2.8", + "nx": "^16.2.1", "prettier": "^2.3.1", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", @@ -23,9 +30,9 @@ }, "scripts": { "preinstall": "node scripts/syncProPackage.js", - "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", - "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh", - "build": "lerna run --stream build", + "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", + "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'", + "build": "yarn nx run-many -t=build", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", @@ -41,10 +48,11 @@ "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream", - "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", + "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream", + "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", + "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", + "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml -f hosting/docker-compose.build.yaml up --build --scale proxy-service=0 ", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", @@ -53,16 +61,16 @@ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", - "build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:pre": "lerna run --stream build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", + "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image", + "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", @@ -101,5 +109,6 @@ "packages/worker", "packages/pro/packages/pro" ] - } + }, + "dependencies": {} } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 6a52cc6c03..9337e689e4 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -88,5 +88,19 @@ "tsconfig-paths": "4.0.0", "typescript": "4.7.3" }, + "nx": { + "targets": { + "build": { + "dependsOn": [ + { + "projects": [ + "@budibase/types" + ], + "target": "build" + } + ] + } + } + }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index de1fc0db5e..de3c45b09d 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -90,5 +90,19 @@ "resolutions": { "loader-utils": "1.4.1" }, + "nx": { + "targets": { + "build": { + "dependsOn": [ + { + "projects": [ + "@budibase/string-templates" + ], + "target": "build" + } + ] + } + } + }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" } diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index e9eb3b3471..427a98f888 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -102,7 +102,9 @@ margin-left: 0; transition: color ease-out 130ms; } - .is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) { + .is-selected:not(.spectrum-ActionButton--emphasized):not( + .spectrum-ActionButton--quiet + ) { background: var(--spectrum-global-color-gray-300); border-color: var(--spectrum-global-color-gray-500); } diff --git a/packages/builder/package.json b/packages/builder/package.json index 95644fc399..01820d8b91 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -9,7 +9,7 @@ "dev:builder": "routify -c dev:vite", "dev:vite": "vite --host 0.0.0.0", "rollup": "rollup -c -w", - "test": "vitest" + "test": "vitest run" }, "jest": { "globals": { @@ -117,5 +117,31 @@ "vite": "^3.0.8", "vitest": "^0.29.2" }, + "nx": { + "targets": { + "build": { + "dependsOn": [ + { + "projects": [ + "@budibase/string-templates", + "@budibase/shared-core" + ], + "target": "build" + } + ] + }, + "test": { + "dependsOn": [ + { + "projects": [ + "@budibase/shared-core", + "@budibase/string-templates" + ], + "target": "build" + } + ] + } + } + }, "gitHead": "115189f72a850bfb52b65ec61d932531bf327072" } diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index dfe30a3711..83e115ebf2 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -32,6 +32,7 @@ - + diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte index 5aa530b028..5b89d92438 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte @@ -4,11 +4,12 @@ export let disabled = false - const { rows, tableId } = getContext("grid") + const { rows, tableId, tableType } = getContext("grid") diff --git a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte index 173fb88829..c020d1a7ac 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte @@ -13,15 +13,18 @@ const dispatch = createEventDispatcher() export let tableId + export let tableType let rows = [] let allValid = false let displayColumn = null + let identifierFields = [] async function importData() { try { await API.importTableData({ tableId, rows, + identifierFields, }) notifications.success("Rows successfully imported") } catch (error) { @@ -45,6 +48,13 @@ - + diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index cb150f71fc..eba17e8ab9 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -1,5 +1,5 @@