2022-11-26 04:01:46 +13:00
|
|
|
import { budibaseTempDir } from "../budibaseDir"
|
|
|
|
import fs from "fs"
|
|
|
|
import { join } from "path"
|
|
|
|
import { context, objectStore } from "@budibase/backend-core"
|
|
|
|
import { ObjectStoreBuckets } from "../../constants"
|
|
|
|
import { updateClientLibrary } from "./clientLibrary"
|
|
|
|
import { checkSlashesInUrl } from "../"
|
|
|
|
import env from "../../environment"
|
|
|
|
import fetch from "node-fetch"
|
2021-03-20 08:07:47 +13:00
|
|
|
const uuid = require("uuid/v4")
|
2022-11-22 07:33:34 +13:00
|
|
|
const tar = require("tar")
|
2021-03-24 06:54:02 +13:00
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
|
|
|
|
export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
|
2022-08-17 21:05:13 +12:00
|
|
|
const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
|
2021-03-20 08:07:47 +13:00
|
|
|
|
|
|
|
/**
|
|
|
|
* The single stack system (Cloud and Builder) should not make use of the file system where possible,
|
|
|
|
* this file handles all of the file access for the system with the intention of limiting it all to one
|
|
|
|
* place. Keeping all of this logic in one place means that when we need to do file system access (like
|
|
|
|
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
|
|
|
|
* be done through an object store instead.
|
|
|
|
*/
|
|
|
|
|
2021-03-24 07:04:53 +13:00
|
|
|
/**
|
|
|
|
* Upon first startup of instance there may not be everything we need in tmp directory, set it up.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function init() {
|
2021-03-24 07:04:53 +13:00
|
|
|
const tempDir = budibaseTempDir()
|
|
|
|
if (!fs.existsSync(tempDir)) {
|
2022-09-13 02:16:31 +12:00
|
|
|
// some test cases fire this quickly enough that
|
|
|
|
// synchronous cases can end up here at the same time
|
|
|
|
try {
|
|
|
|
fs.mkdirSync(tempDir)
|
2022-11-26 04:01:46 +13:00
|
|
|
} catch (err: any) {
|
2022-09-13 02:16:31 +12:00
|
|
|
if (!err || err.code !== "EEXIST") {
|
|
|
|
throw err
|
|
|
|
}
|
|
|
|
}
|
2021-03-24 07:04:53 +13:00
|
|
|
}
|
2021-03-26 07:03:58 +13:00
|
|
|
const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
|
|
|
|
if (env.isTest() && !fs.existsSync(clientLibPath)) {
|
|
|
|
fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
|
|
|
|
}
|
2021-03-24 07:04:53 +13:00
|
|
|
}
|
|
|
|
|
2021-03-20 08:07:47 +13:00
|
|
|
/**
|
|
|
|
* Checks if the system is currently in development mode and if it is makes sure
|
|
|
|
* everything required to function is ready.
|
|
|
|
*/
|
|
|
|
exports.checkDevelopmentEnvironment = () => {
|
2022-01-29 07:52:34 +13:00
|
|
|
if (!env.isDev() || env.isTest()) {
|
2021-03-24 06:54:02 +13:00
|
|
|
return
|
|
|
|
}
|
|
|
|
if (!fs.existsSync(budibaseTempDir())) {
|
2021-04-02 04:36:27 +13:00
|
|
|
fs.mkdirSync(budibaseTempDir())
|
2021-03-24 06:54:02 +13:00
|
|
|
}
|
2021-04-02 04:36:27 +13:00
|
|
|
let error
|
2021-03-24 06:54:02 +13:00
|
|
|
if (!fs.existsSync(join(process.cwd(), ".env"))) {
|
|
|
|
error = "Must run via yarn once to generate environment."
|
|
|
|
}
|
|
|
|
if (error) {
|
|
|
|
console.error(error)
|
2021-03-20 08:07:47 +13:00
|
|
|
process.exit(-1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Used to retrieve a handlebars file from the system which will be used as a template.
|
|
|
|
* This is allowable as the template handlebars files should be static and identical across
|
|
|
|
* the cluster.
|
|
|
|
* @param {string} path The path to the handlebars file which is to be loaded.
|
|
|
|
* @returns {string} The loaded handlebars file as a string - loaded as utf8.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function loadHandlebarsFile(path: string) {
|
2021-03-20 08:07:47 +13:00
|
|
|
return fs.readFileSync(path, "utf8")
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* When return a file from the API need to write the file to the system temporarily so we
|
|
|
|
* can create a read stream to send.
|
|
|
|
* @param {string} contents the contents of the file which is to be returned from the API.
|
2022-11-30 05:03:22 +13:00
|
|
|
* @param {string} encoding the encoding of the file to return (utf8 default)
|
2021-03-20 08:07:47 +13:00
|
|
|
* @return {Object} the read stream which can be put into the koa context body.
|
|
|
|
*/
|
2022-11-30 05:03:22 +13:00
|
|
|
export function apiFileReturn(
|
|
|
|
contents: string,
|
|
|
|
encoding: BufferEncoding = "utf8"
|
|
|
|
) {
|
2021-03-20 08:07:47 +13:00
|
|
|
const path = join(budibaseTempDir(), uuid())
|
2022-11-30 05:03:22 +13:00
|
|
|
fs.writeFileSync(path, contents, { encoding })
|
|
|
|
return fs.createReadStream(path, { encoding })
|
2021-03-20 08:07:47 +13:00
|
|
|
}
|
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
export function streamFile(path: string) {
|
2022-10-12 06:21:58 +13:00
|
|
|
return fs.createReadStream(path)
|
|
|
|
}
|
|
|
|
|
2021-09-28 07:12:41 +13:00
|
|
|
/**
|
|
|
|
* Writes the provided contents to a temporary file, which can be used briefly.
|
|
|
|
* @param {string} fileContents contents which will be written to a temp file.
|
|
|
|
* @return {string} the path to the temp file.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function storeTempFile(fileContents: string) {
|
2021-09-28 07:12:41 +13:00
|
|
|
const path = join(budibaseTempDir(), uuid())
|
|
|
|
fs.writeFileSync(path, fileContents)
|
|
|
|
return path
|
|
|
|
}
|
|
|
|
|
2021-09-29 06:05:52 +13:00
|
|
|
/**
|
|
|
|
* Utility function for getting a file read stream - a simple in memory buffered read
|
|
|
|
* stream doesn't work for pouchdb.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function stringToFileStream(contents: string) {
|
2021-09-29 06:05:52 +13:00
|
|
|
const path = exports.storeTempFile(contents)
|
|
|
|
return fs.createReadStream(path)
|
|
|
|
}
|
|
|
|
|
2021-09-28 07:12:41 +13:00
|
|
|
/**
|
|
|
|
* Creates a temp file and returns it from the API.
|
|
|
|
* @param {string} fileContents the contents to be returned in file.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function sendTempFile(fileContents: string) {
|
2021-09-28 07:12:41 +13:00
|
|
|
const path = exports.storeTempFile(fileContents)
|
|
|
|
return fs.createReadStream(path)
|
|
|
|
}
|
|
|
|
|
2021-03-24 06:54:02 +13:00
|
|
|
/**
|
2021-07-08 23:56:54 +12:00
|
|
|
* Uploads the latest client library to the object store.
|
2021-03-24 06:54:02 +13:00
|
|
|
* @param {string} appId The ID of the app which is being created.
|
|
|
|
* @return {Promise<void>} once promise completes app resources should be ready in object store.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function createApp(appId: string) {
|
2021-07-08 23:56:54 +12:00
|
|
|
await updateClientLibrary(appId)
|
2021-03-20 08:07:47 +13:00
|
|
|
}
|
|
|
|
|
2021-03-24 06:54:02 +13:00
|
|
|
/**
|
|
|
|
* Removes all of the assets created for an app in the object store.
|
|
|
|
* @param {string} appId The ID of the app which is being deleted.
|
|
|
|
* @return {Promise<void>} once promise completes the app resources will be removed from object store.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function deleteApp(appId: string) {
|
|
|
|
await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
|
2021-03-20 08:07:47 +13:00
|
|
|
}
|
|
|
|
|
2021-03-24 06:54:02 +13:00
|
|
|
/**
|
|
|
|
* Retrieves a template and pipes it to minio as well as making it available temporarily.
|
|
|
|
* @param {string} type The type of template which is to be retrieved.
|
|
|
|
* @param name
|
|
|
|
* @return {Promise<*>}
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function downloadTemplate(type: string, name: string) {
|
2021-03-20 08:07:47 +13:00
|
|
|
const DEFAULT_TEMPLATES_BUCKET =
|
|
|
|
"prod-budi-templates.s3-eu-west-1.amazonaws.com"
|
|
|
|
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
|
2022-11-26 04:01:46 +13:00
|
|
|
return objectStore.downloadTarball(
|
|
|
|
templateUrl,
|
|
|
|
ObjectStoreBuckets.TEMPLATES,
|
|
|
|
type
|
|
|
|
)
|
2021-03-20 08:07:47 +13:00
|
|
|
}
|
2021-03-23 06:19:45 +13:00
|
|
|
|
2021-03-24 06:54:02 +13:00
|
|
|
/**
|
|
|
|
* Retrieves component libraries from object store (or tmp symlink if in local)
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function getComponentLibraryManifest(library: string) {
|
2022-11-22 07:33:34 +13:00
|
|
|
const appId = context.getAppId()
|
2021-03-26 03:46:32 +13:00
|
|
|
const filename = "manifest.json"
|
|
|
|
/* istanbul ignore next */
|
2021-04-07 22:31:19 +12:00
|
|
|
// when testing in cypress and so on we need to get the package
|
|
|
|
// as the environment may not be fully fleshed out for dev or prod
|
|
|
|
if (env.isTest()) {
|
2021-09-07 21:05:24 +12:00
|
|
|
library = library.replace("standard-components", "client")
|
2021-04-07 22:31:19 +12:00
|
|
|
const lib = library.split("/")[1]
|
|
|
|
const path = require.resolve(library).split(lib)[0]
|
|
|
|
return require(join(path, lib, filename))
|
|
|
|
} else if (env.isDev()) {
|
2021-09-01 22:41:48 +12:00
|
|
|
const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
|
2021-04-02 04:28:51 +13:00
|
|
|
// always load from new so that updates are refreshed
|
|
|
|
delete require.cache[require.resolve(path)]
|
|
|
|
return require(path)
|
2021-03-26 02:32:05 +13:00
|
|
|
}
|
2021-07-08 04:07:42 +12:00
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
if (!appId) {
|
|
|
|
throw new Error("No app ID found - cannot get component libraries")
|
|
|
|
}
|
|
|
|
|
2021-07-08 04:07:42 +12:00
|
|
|
let resp
|
2022-08-25 22:07:35 +12:00
|
|
|
let path
|
2021-07-08 04:07:42 +12:00
|
|
|
try {
|
|
|
|
// Try to load the manifest from the new file location
|
2022-08-25 22:07:35 +12:00
|
|
|
path = join(appId, filename)
|
2022-11-26 04:01:46 +13:00
|
|
|
resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
|
2021-07-08 04:07:42 +12:00
|
|
|
} catch (error) {
|
2022-08-25 22:07:35 +12:00
|
|
|
console.error(
|
|
|
|
`component-manifest-objectstore=failed appId=${appId} path=${path}`,
|
|
|
|
error
|
|
|
|
)
|
2021-07-08 04:07:42 +12:00
|
|
|
// Fallback to loading it from the old location for old apps
|
2022-08-25 22:07:35 +12:00
|
|
|
path = join(appId, "node_modules", library, "package", filename)
|
2022-11-26 04:01:46 +13:00
|
|
|
resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
|
2021-07-08 04:07:42 +12:00
|
|
|
}
|
2021-03-24 06:54:02 +13:00
|
|
|
if (typeof resp !== "string") {
|
|
|
|
resp = resp.toString("utf8")
|
|
|
|
}
|
|
|
|
return JSON.parse(resp)
|
|
|
|
}
|
|
|
|
|
2021-03-23 06:19:45 +13:00
|
|
|
/**
|
|
|
|
* All file reads come through here just to make sure all of them make sense
|
|
|
|
* allows a centralised location to check logic is all good.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function readFileSync(
|
|
|
|
filepath: string,
|
|
|
|
options: BufferEncoding = "utf8"
|
|
|
|
) {
|
|
|
|
return fs.readFileSync(filepath, { encoding: options })
|
2021-03-23 06:19:45 +13:00
|
|
|
}
|
2021-03-23 07:06:10 +13:00
|
|
|
|
2021-03-26 03:46:32 +13:00
|
|
|
/**
|
|
|
|
* Given a set of app IDs makes sure file system is cleared of any of their temp info.
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function cleanup(appIds: string[]) {
|
2021-03-26 03:46:32 +13:00
|
|
|
for (let appId of appIds) {
|
2021-07-07 20:34:40 +12:00
|
|
|
const path = join(budibaseTempDir(), appId)
|
|
|
|
if (fs.existsSync(path)) {
|
|
|
|
fs.rmdirSync(path, { recursive: true })
|
|
|
|
}
|
2021-03-26 03:46:32 +13:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
export function createTempFolder(item: string) {
|
2022-09-07 03:28:35 +12:00
|
|
|
const path = join(budibaseTempDir(), item)
|
2022-09-02 07:04:45 +12:00
|
|
|
try {
|
2022-09-07 03:28:35 +12:00
|
|
|
// remove old tmp directories automatically - don't combine
|
2022-09-02 07:04:45 +12:00
|
|
|
if (fs.existsSync(path)) {
|
|
|
|
fs.rmSync(path, { recursive: true, force: true })
|
|
|
|
}
|
|
|
|
fs.mkdirSync(path)
|
2022-11-26 04:01:46 +13:00
|
|
|
} catch (err: any) {
|
2022-09-07 03:28:35 +12:00
|
|
|
throw new Error(`Path cannot be created: ${err.message}`)
|
2022-09-02 07:04:45 +12:00
|
|
|
}
|
2022-08-31 08:37:08 +12:00
|
|
|
|
2022-09-07 03:28:35 +12:00
|
|
|
return path
|
|
|
|
}
|
2022-08-31 08:37:08 +12:00
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function extractTarball(fromFilePath: string, toPath: string) {
|
2022-09-07 03:28:35 +12:00
|
|
|
await tar.extract({
|
|
|
|
file: fromFilePath,
|
|
|
|
C: toPath,
|
|
|
|
})
|
2022-08-31 08:37:08 +12:00
|
|
|
}
|
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function getPluginMetadata(path: string) {
|
|
|
|
let metadata: { schema?: any; package?: any } = {}
|
2022-08-11 07:01:48 +12:00
|
|
|
try {
|
|
|
|
const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
|
|
|
|
const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
|
2022-09-07 03:28:35 +12:00
|
|
|
|
2022-08-11 07:01:48 +12:00
|
|
|
metadata.schema = JSON.parse(schema)
|
|
|
|
metadata.package = JSON.parse(pkg)
|
2022-09-07 03:28:35 +12:00
|
|
|
|
|
|
|
if (
|
|
|
|
!metadata.package.name ||
|
|
|
|
!metadata.package.version ||
|
|
|
|
!metadata.package.description
|
|
|
|
) {
|
|
|
|
throw new Error(
|
|
|
|
"package.json is missing one of 'name', 'version' or 'description'."
|
|
|
|
)
|
|
|
|
}
|
2022-11-26 04:01:46 +13:00
|
|
|
} catch (err: any) {
|
2022-09-07 03:28:35 +12:00
|
|
|
throw new Error(
|
|
|
|
`Unable to process schema.json/package.json in plugin. ${err.message}`
|
|
|
|
)
|
2022-08-11 07:01:48 +12:00
|
|
|
}
|
2022-08-31 08:37:08 +12:00
|
|
|
|
2022-08-11 07:01:48 +12:00
|
|
|
return { metadata, directory: path }
|
|
|
|
}
|
|
|
|
|
2022-11-26 04:01:46 +13:00
|
|
|
export async function getDatasourcePlugin(
|
|
|
|
name: string,
|
|
|
|
url: string,
|
|
|
|
hash: string
|
|
|
|
) {
|
2022-08-17 21:05:13 +12:00
|
|
|
if (!fs.existsSync(DATASOURCE_PATH)) {
|
|
|
|
fs.mkdirSync(DATASOURCE_PATH)
|
|
|
|
}
|
|
|
|
const filename = join(DATASOURCE_PATH, name)
|
2022-08-19 02:21:55 +12:00
|
|
|
const metadataName = `${filename}.bbmetadata`
|
2022-08-17 21:05:13 +12:00
|
|
|
if (fs.existsSync(filename)) {
|
2022-08-19 02:21:55 +12:00
|
|
|
const currentHash = fs.readFileSync(metadataName, "utf8")
|
|
|
|
// if hash is the same return the file, otherwise remove it and re-download
|
|
|
|
if (currentHash === hash) {
|
|
|
|
return require(filename)
|
|
|
|
} else {
|
2022-08-19 05:23:07 +12:00
|
|
|
console.log(`Updating plugin: ${name}`)
|
2022-09-16 22:25:28 +12:00
|
|
|
delete require.cache[require.resolve(filename)]
|
2022-08-19 02:21:55 +12:00
|
|
|
fs.unlinkSync(filename)
|
|
|
|
}
|
2022-08-17 21:05:13 +12:00
|
|
|
}
|
2022-08-18 10:13:51 +12:00
|
|
|
const fullUrl = checkSlashesInUrl(
|
|
|
|
`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`
|
|
|
|
)
|
|
|
|
const response = await fetch(fullUrl)
|
2022-08-17 21:05:13 +12:00
|
|
|
if (response.status === 200) {
|
|
|
|
const content = await response.text()
|
|
|
|
fs.writeFileSync(filename, content)
|
2022-08-19 02:21:55 +12:00
|
|
|
fs.writeFileSync(metadataName, hash)
|
2022-09-13 23:16:00 +12:00
|
|
|
return require(filename)
|
2022-08-17 21:05:13 +12:00
|
|
|
} else {
|
|
|
|
throw new Error(
|
|
|
|
`Unable to retrieve plugin - reason: ${await response.text()}`
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-07 03:28:35 +12:00
|
|
|
/**
|
|
|
|
* Find for a file recursively from start path applying filter, return first match
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function findFileRec(startPath: string, filter: any) {
|
2022-09-07 03:28:35 +12:00
|
|
|
if (!fs.existsSync(startPath)) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-09-13 03:21:47 +12:00
|
|
|
const files = fs.readdirSync(startPath)
|
2022-09-07 03:28:35 +12:00
|
|
|
for (let i = 0, len = files.length; i < len; i++) {
|
|
|
|
const filename = join(startPath, files[i])
|
|
|
|
const stat = fs.lstatSync(filename)
|
|
|
|
|
|
|
|
if (stat.isDirectory()) {
|
2022-09-13 03:21:47 +12:00
|
|
|
return exports.findFileRec(filename, filter)
|
2022-09-07 03:28:35 +12:00
|
|
|
} else if (filename.endsWith(filter)) {
|
|
|
|
return filename
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Remove a folder which is not empty from the file system
|
|
|
|
*/
|
2022-11-26 04:01:46 +13:00
|
|
|
export function deleteFolderFileSystem(path: string) {
|
2022-09-07 03:28:35 +12:00
|
|
|
if (!fs.existsSync(path)) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
fs.rmSync(path, { recursive: true, force: true })
|
|
|
|
}
|