Adding a deployment service which takes over from the lambdas in local operation, this may become part of the hosting portal if we ever decide to opensource that part of it.
This commit is contained in:
parent
86b0c4963c
commit
15f8328770
1
hosting/deployment
Symbolic link
1
hosting/deployment
Symbolic link
|
@ -0,0 +1 @@
|
|||
../packages/deployment/
|
|
@ -16,13 +16,30 @@ services:
|
|||
BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
|
||||
LOGO_URL: ${LOGO_URL}
|
||||
PORT: ${APP_PORT}
|
||||
depends_on:
|
||||
- deployment-service
|
||||
|
||||
deployment-service:
|
||||
build: ./deployment
|
||||
ports:
|
||||
- "${DEPLOYMENT_PORT}:${DEPLOYMENT_PORT}"
|
||||
environment:
|
||||
SELF_HOSTED: 1,
|
||||
DEPLOYMENT_API_KEY: ${DEPLOYMENT_API_KEY}
|
||||
PORT: ${DEPLOYMENT_PORT}
|
||||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
|
||||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
|
||||
RAW_MINIO_URL: http://nginx-service:${MINIO_PORT}
|
||||
COUCH_DB_USERNAME: ${COUCH_DB_USER}
|
||||
COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
|
||||
RAW_COUCH_DB_URL: http://couchdb-service:5984
|
||||
depends_on:
|
||||
- nginx-service
|
||||
- minio-service
|
||||
- couch-init
|
||||
|
||||
minio-service:
|
||||
image: minio/minio:RELEASE.2020-12-10T01-54-29Z
|
||||
image: minio/minio
|
||||
volumes:
|
||||
- data1:/data
|
||||
ports:
|
||||
|
|
|
@ -2,9 +2,11 @@ MINIO_ACCESS_KEY=budibase
|
|||
MINIO_SECRET_KEY=budibase
|
||||
COUCH_DB_PASSWORD=budibase
|
||||
COUCH_DB_USER=budibase
|
||||
DEPLOYMENT_API_KEY=budibase
|
||||
BUDIBASE_ENVIRONMENT=PRODUCTION
|
||||
HOSTING_URL="http://localhost:4001"
|
||||
LOGO_URL=https://logoipsum.com/logo/logo-15.svg
|
||||
APP_PORT=4002
|
||||
MINIO_PORT=4003
|
||||
COUCH_DB_PORT=4004
|
||||
DEPLOYMENT_PORT=4006
|
||||
|
|
|
@ -40,7 +40,7 @@
|
|||
<Toggle thin text="Self hosted" bind:checked={selfhosted} />
|
||||
{#if selfhosted}
|
||||
<Input bind:value={hostingInfo.appServerUrl} label="Apps URL" />
|
||||
<Input bind:value={hostingInfo.objectStoreUrl} label="Object store URL" />
|
||||
<Input bind:value={hostingInfo.deploymentServerUrl} label="Deployments URL" />
|
||||
<Toggle thin text="HTTPS" bind:checked={hostingInfo.useHttps} />
|
||||
{/if}
|
||||
</ModalContent>
|
||||
|
|
2
packages/deployment/.gitignore
vendored
Normal file
2
packages/deployment/.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
node_modules/
|
||||
.env
|
15
packages/deployment/Dockerfile
Normal file
15
packages/deployment/Dockerfile
Normal file
|
@ -0,0 +1,15 @@
|
|||
FROM node:12-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# copy files and install dependencies
|
||||
COPY . ./
|
||||
RUN yarn
|
||||
|
||||
EXPOSE 4001
|
||||
|
||||
# have to add node environment production after install
|
||||
# due to this causing yarn to stop installing dev dependencies
|
||||
# which are actually needed to get this environment up and running
|
||||
ENV NODE_ENV=production
|
||||
CMD ["yarn", "run:docker"]
|
34
packages/deployment/package.json
Normal file
34
packages/deployment/package.json
Normal file
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"name": "@budibase/deployment",
|
||||
"email": "hi@budibase.com",
|
||||
"version": "0.3.8",
|
||||
"description": "Budibase Deployment Server",
|
||||
"main": "src/index.js",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/Budibase/budibase.git"
|
||||
},
|
||||
"keywords": [
|
||||
"budibase"
|
||||
],
|
||||
"scripts": {
|
||||
"run:docker": "node src/index.js"
|
||||
},
|
||||
"author": "Budibase",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@koa/router": "^8.0.0",
|
||||
"aws-sdk": "^2.811.0",
|
||||
"got": "^11.8.1",
|
||||
"joi": "^17.2.1",
|
||||
"koa": "^2.7.0",
|
||||
"koa-body": "^4.2.0",
|
||||
"koa-compress": "^4.0.1",
|
||||
"koa-pino-logger": "^3.0.0",
|
||||
"koa-send": "^5.0.0",
|
||||
"koa-session": "^5.12.0",
|
||||
"koa-static": "^5.0.0",
|
||||
"pino-pretty": "^4.0.0",
|
||||
"server-destroy": "^1.0.1"
|
||||
}
|
||||
}
|
86
packages/deployment/src/api/controllers/deploy.js
Normal file
86
packages/deployment/src/api/controllers/deploy.js
Normal file
|
@ -0,0 +1,86 @@
|
|||
const env = require("../../environment")
|
||||
const got = require("got")
|
||||
const AWS = require("aws-sdk")
|
||||
|
||||
const APP_BUCKET = "app-assets"
|
||||
// this doesn't matter in self host
|
||||
const REGION = "eu-west-1"
|
||||
|
||||
async function getCouchSession() {
|
||||
// fetch session token for the api user
|
||||
const session = await got.post(`${env.RAW_COUCH_DB_URL}/_session`, {
|
||||
responseType: "json",
|
||||
json: {
|
||||
username: env.COUCH_DB_USERNAME,
|
||||
password: env.COUCH_DB_PASSWORD,
|
||||
}
|
||||
})
|
||||
|
||||
const cookie = session.headers["set-cookie"][0]
|
||||
// Get the session cookie value only
|
||||
return cookie.split(";")[0]
|
||||
}
|
||||
|
||||
async function getMinioSession() {
|
||||
AWS.config.update({
|
||||
accessKeyId: env.MINIO_ACCESS_KEY,
|
||||
secretAccessKey: env.MINIO_SECRET_KEY,
|
||||
})
|
||||
|
||||
// make sure the bucket exists
|
||||
const objClient = new AWS.S3({
|
||||
endpoint: env.RAW_MINIO_URL,
|
||||
region: REGION,
|
||||
s3ForcePathStyle: true, // needed with minio?
|
||||
params: {
|
||||
Bucket: APP_BUCKET,
|
||||
},
|
||||
})
|
||||
// make sure the bucket exists
|
||||
try {
|
||||
await objClient.headBucket({ Bucket: APP_BUCKET }).promise()
|
||||
} catch (err) {
|
||||
// bucket doesn't exist create it
|
||||
if (err.statusCode === 404) {
|
||||
await objClient.createBucket({ Bucket: APP_BUCKET }).promise()
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
// TODO: this doesn't seem to work get an error
|
||||
// TODO: Generating temporary credentials not allowed for this request.
|
||||
// TODO: this should work based on minio documentation
|
||||
// const sts = new AWS.STS({
|
||||
// endpoint: env.RAW_MINIO_URL,
|
||||
// region: REGION,
|
||||
// s3ForcePathStyle: true,
|
||||
// })
|
||||
// // NOTE: In the following commands RoleArn and RoleSessionName are not meaningful for MinIO
|
||||
// const params = {
|
||||
// DurationSeconds: 3600,
|
||||
// ExternalId: "123ABC",
|
||||
// Policy: '{"Version":"2012-10-17","Statement":[{"Sid":"Stmt1","Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}]}',
|
||||
// RoleArn: 'arn:xxx:xxx:xxx:xxxx',
|
||||
// RoleSessionName: 'anything',
|
||||
// };
|
||||
// const assumedRole = await sts.assumeRole(params).promise();
|
||||
// if (!assumedRole) {
|
||||
// throw "Unable to get access to object store."
|
||||
// }
|
||||
// return assumedRole.Credentials
|
||||
// TODO: need to do something better than this
|
||||
return {
|
||||
accessKeyId: env.MINIO_ACCESS_KEY,
|
||||
secretAccessKey: env.MINIO_SECRET_KEY,
|
||||
}
|
||||
}
|
||||
|
||||
exports.deploy = async ctx => {
|
||||
ctx.body = {
|
||||
couchDbSession: await getCouchSession(),
|
||||
bucket: APP_BUCKET,
|
||||
objectStoreSession: await getMinioSession(),
|
||||
couchDbUrl: env.RAW_COUCH_DB_URL,
|
||||
objectStoreUrl: env.RAW_MINIO_URL,
|
||||
}
|
||||
}
|
45
packages/deployment/src/api/index.js
Normal file
45
packages/deployment/src/api/index.js
Normal file
|
@ -0,0 +1,45 @@
|
|||
const Router = require("@koa/router")
|
||||
const compress = require("koa-compress")
|
||||
const zlib = require("zlib")
|
||||
const { routes } = require("./routes")
|
||||
|
||||
const router = new Router()
|
||||
|
||||
router
|
||||
.use(
|
||||
compress({
|
||||
threshold: 2048,
|
||||
gzip: {
|
||||
flush: zlib.Z_SYNC_FLUSH,
|
||||
},
|
||||
deflate: {
|
||||
flush: zlib.Z_SYNC_FLUSH,
|
||||
},
|
||||
br: false,
|
||||
})
|
||||
)
|
||||
.use("/health", ctx => (ctx.status = 200))
|
||||
|
||||
// error handling middleware
|
||||
router.use(async (ctx, next) => {
|
||||
try {
|
||||
await next()
|
||||
} catch (err) {
|
||||
ctx.log.error(err)
|
||||
ctx.status = err.status || err.statusCode || 500
|
||||
ctx.body = {
|
||||
message: err.message,
|
||||
status: ctx.status,
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
router.get("/health", ctx => (ctx.status = 200))
|
||||
|
||||
// authenticated routes
|
||||
for (let route of routes) {
|
||||
router.use(route.routes())
|
||||
router.use(route.allowedMethods())
|
||||
}
|
||||
|
||||
module.exports = router
|
10
packages/deployment/src/api/routes/deploy.js
Normal file
10
packages/deployment/src/api/routes/deploy.js
Normal file
|
@ -0,0 +1,10 @@
|
|||
const Router = require("@koa/router")
|
||||
const controller = require("../controllers/deploy")
|
||||
const checkKey = require("../../middleware/check-key")
|
||||
|
||||
const router = Router()
|
||||
|
||||
router
|
||||
.post("/api/deploy", checkKey, controller.deploy)
|
||||
|
||||
module.exports = router
|
5
packages/deployment/src/api/routes/index.js
Normal file
5
packages/deployment/src/api/routes/index.js
Normal file
|
@ -0,0 +1,5 @@
|
|||
const deployRoutes = require("./deploy")
|
||||
|
||||
exports.routes = [
|
||||
deployRoutes,
|
||||
]
|
15
packages/deployment/src/environment.js
Normal file
15
packages/deployment/src/environment.js
Normal file
|
@ -0,0 +1,15 @@
|
|||
module.exports = {
|
||||
SELF_HOSTED: process.env.SELF_HOSTED,
|
||||
DEPLOYMENT_API_KEY: process.env.DEPLOYMENT_API_KEY,
|
||||
PORT: process.env.PORT,
|
||||
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
|
||||
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
|
||||
RAW_MINIO_URL: process.env.RAW_MINIO_URL,
|
||||
COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME,
|
||||
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
|
||||
RAW_COUCH_DB_URL: process.env.RAW_COUCH_DB_URL,
|
||||
_set(key, value) {
|
||||
process.env[key] = value
|
||||
module.exports[key] = value
|
||||
},
|
||||
}
|
48
packages/deployment/src/index.js
Normal file
48
packages/deployment/src/index.js
Normal file
|
@ -0,0 +1,48 @@
|
|||
const Koa = require("koa")
|
||||
const destroyable = require("server-destroy")
|
||||
const koaBody = require("koa-body")
|
||||
const logger = require("koa-pino-logger")
|
||||
const http = require("http")
|
||||
const api = require("./api")
|
||||
const env = require("./environment")
|
||||
|
||||
const app = new Koa()
|
||||
|
||||
if (!env.SELF_HOSTED) {
|
||||
throw "Currently this service only supports use in self hosting"
|
||||
}
|
||||
|
||||
// set up top level koa middleware
|
||||
app.use(koaBody({ multipart: true }))
|
||||
|
||||
app.use(
|
||||
logger({
|
||||
prettyPrint: {
|
||||
levelFirst: true,
|
||||
},
|
||||
level: env.LOG_LEVEL || "error",
|
||||
})
|
||||
)
|
||||
|
||||
// api routes
|
||||
app.use(api.routes())
|
||||
|
||||
const server = http.createServer(app.callback())
|
||||
destroyable(server)
|
||||
|
||||
server.on("close", () => console.log("Server Closed"))
|
||||
|
||||
module.exports = server.listen(env.PORT || 4002, async () => {
|
||||
console.log(`Deployment running on ${JSON.stringify(server.address())}`)
|
||||
})
|
||||
|
||||
process.on("uncaughtException", err => {
|
||||
console.error(err)
|
||||
server.close()
|
||||
server.destroy()
|
||||
})
|
||||
|
||||
process.on("SIGTERM", () => {
|
||||
server.close()
|
||||
server.destroy()
|
||||
})
|
4
packages/deployment/src/middleware/check-key.js
Normal file
4
packages/deployment/src/middleware/check-key.js
Normal file
|
@ -0,0 +1,4 @@
|
|||
module.exports = async (ctx, next) => {
|
||||
// TODO: need to check the API key provided in the header
|
||||
await next()
|
||||
}
|
1166
packages/deployment/yarn.lock
Normal file
1166
packages/deployment/yarn.lock
Normal file
File diff suppressed because it is too large
Load diff
|
@ -1,9 +1,11 @@
|
|||
const AWS = require("aws-sdk")
|
||||
const fetch = require("node-fetch")
|
||||
const env = require("../../../environment")
|
||||
const { deployToObjectStore, performReplication } = require("./utils")
|
||||
const CouchDB = require("pouchdb")
|
||||
const PouchDB = require("../../../db")
|
||||
const {
|
||||
deployToObjectStore,
|
||||
performReplication,
|
||||
fetchCredentials,
|
||||
} = require("./utils")
|
||||
|
||||
/**
|
||||
* Verifies the users API key and
|
||||
|
@ -12,26 +14,12 @@ const PouchDB = require("../../../db")
|
|||
* @param {object} deployment - information about the active deployment, including the appId and quota.
|
||||
*/
|
||||
exports.preDeployment = async function(deployment) {
|
||||
const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
apiKey: env.BUDIBASE_API_KEY,
|
||||
appId: deployment.getAppId(),
|
||||
quota: deployment.getQuota(),
|
||||
}),
|
||||
const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, {
|
||||
apiKey: env.BUDIBASE_API_KEY,
|
||||
appId: deployment.getAppId(),
|
||||
quota: deployment.getQuota(),
|
||||
})
|
||||
|
||||
const json = await response.json()
|
||||
if (json.errors) {
|
||||
throw new Error(json.errors)
|
||||
}
|
||||
|
||||
if (response.status !== 200) {
|
||||
throw new Error(
|
||||
`Error fetching temporary credentials for api key: ${env.BUDIBASE_API_KEY}`
|
||||
)
|
||||
}
|
||||
|
||||
// set credentials here, means any time we're verified we're ready to go
|
||||
if (json.credentials) {
|
||||
AWS.config.update({
|
||||
|
@ -88,14 +76,10 @@ exports.deploy = async function(deployment) {
|
|||
|
||||
exports.replicateDb = async function(deployment) {
|
||||
const appId = deployment.getAppId()
|
||||
const { session } = deployment.getVerification()
|
||||
const localDb = new PouchDB(appId)
|
||||
const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, {
|
||||
fetch: function(url, opts) {
|
||||
opts.headers.set("Cookie", `${session};`)
|
||||
return CouchDB.fetch(url, opts)
|
||||
},
|
||||
})
|
||||
|
||||
return performReplication(localDb, remoteDb)
|
||||
const verification = deployment.getVerification()
|
||||
return performReplication(
|
||||
appId,
|
||||
verification.couchDbSession,
|
||||
env.DEPLOYMENT_DB_URL
|
||||
)
|
||||
}
|
||||
|
|
|
@ -1,16 +1,31 @@
|
|||
const env = require("../../../environment")
|
||||
const AWS = require("aws-sdk")
|
||||
const { deployToObjectStore, performReplication } = require("./utils")
|
||||
const CouchDB = require("pouchdb")
|
||||
const PouchDB = require("../../../db")
|
||||
|
||||
const APP_BUCKET = "app-assets"
|
||||
const {
|
||||
deployToObjectStore,
|
||||
performReplication,
|
||||
fetchCredentials,
|
||||
} = require("./utils")
|
||||
const { getDeploymentUrl } = require("../../../utilities/builder/hosting")
|
||||
const { join } = require("path")
|
||||
|
||||
exports.preDeployment = async function() {
|
||||
AWS.config.update({
|
||||
accessKeyId: env.MINIO_ACCESS_KEY,
|
||||
secretAccessKey: env.MINIO_SECRET_KEY,
|
||||
const url = join(await getDeploymentUrl(), "api", "deploy")
|
||||
const json = await fetchCredentials(url, {
|
||||
apiKey: env.BUDIBASE_API_KEY,
|
||||
})
|
||||
|
||||
// response contains:
|
||||
// couchDbSession, bucket, objectStoreSession, couchDbUrl, objectStoreUrl
|
||||
|
||||
// set credentials here, means any time we're verified we're ready to go
|
||||
if (json.objectStoreSession) {
|
||||
AWS.config.update({
|
||||
accessKeyId: json.objectStoreSession.AccessKeyId,
|
||||
secretAccessKey: json.objectStoreSession.SecretAccessKey,
|
||||
})
|
||||
}
|
||||
|
||||
return json
|
||||
}
|
||||
|
||||
exports.postDeployment = async function() {
|
||||
|
@ -19,25 +34,15 @@ exports.postDeployment = async function() {
|
|||
|
||||
exports.deploy = async function(deployment) {
|
||||
const appId = deployment.getAppId()
|
||||
var objClient = new AWS.S3({
|
||||
endpoint: env.MINIO_URL,
|
||||
const verification = deployment.getVerification()
|
||||
const objClient = new AWS.S3({
|
||||
endpoint: verification.objectStoreUrl,
|
||||
s3ForcePathStyle: true, // needed with minio?
|
||||
signatureVersion: "v4",
|
||||
params: {
|
||||
Bucket: APP_BUCKET,
|
||||
Bucket: verification.bucket,
|
||||
},
|
||||
})
|
||||
// checking the bucket exists
|
||||
try {
|
||||
await objClient.headBucket({ Bucket: APP_BUCKET }).promise()
|
||||
} catch (err) {
|
||||
// bucket doesn't exist create it
|
||||
if (err.statusCode === 404) {
|
||||
await objClient.createBucket({ Bucket: APP_BUCKET }).promise()
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
// no metadata, aws has account ID in metadata
|
||||
const metadata = {}
|
||||
await deployToObjectStore(appId, objClient, metadata)
|
||||
|
@ -45,8 +50,10 @@ exports.deploy = async function(deployment) {
|
|||
|
||||
exports.replicateDb = async function(deployment) {
|
||||
const appId = deployment.getAppId()
|
||||
const localDb = new PouchDB(appId)
|
||||
|
||||
const remoteDb = new CouchDB(`${env.COUCH_DB_URL}/${appId}`)
|
||||
return performReplication(localDb, remoteDb)
|
||||
const verification = deployment.getVerification()
|
||||
return performReplication(
|
||||
appId,
|
||||
verification.couchDbSession,
|
||||
verification.couchDbUrl
|
||||
)
|
||||
}
|
||||
|
|
|
@ -4,6 +4,7 @@ const { walkDir } = require("../../../utilities")
|
|||
const { join } = require("../../../utilities/centralPath")
|
||||
const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
|
||||
const PouchDB = require("../../../db")
|
||||
const CouchDB = require("pouchdb")
|
||||
|
||||
const CONTENT_TYPE_MAP = {
|
||||
html: "text/html",
|
||||
|
@ -11,6 +12,26 @@ const CONTENT_TYPE_MAP = {
|
|||
js: "application/javascript",
|
||||
}
|
||||
|
||||
exports.fetchCredentials = async function(url, body) {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
const json = await response.json()
|
||||
if (json.errors) {
|
||||
throw new Error(json.errors)
|
||||
}
|
||||
|
||||
if (response.status !== 200) {
|
||||
throw new Error(
|
||||
`Error fetching temporary credentials for api key: ${body.apiKey}`
|
||||
)
|
||||
}
|
||||
|
||||
return json
|
||||
}
|
||||
|
||||
exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
|
||||
const extension = [...file.name.split(".")].pop()
|
||||
const fileBytes = fs.readFileSync(file.path)
|
||||
|
@ -89,8 +110,17 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
|
|||
}
|
||||
}
|
||||
|
||||
exports.performReplication = (local, remote) => {
|
||||
exports.performReplication = (appId, session, dbUrl) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const local = new PouchDB(appId)
|
||||
|
||||
const remote = new CouchDB(`${dbUrl}/${appId}`, {
|
||||
fetch: function(url, opts) {
|
||||
opts.headers.set("Cookie", `${session};`)
|
||||
return CouchDB.fetch(url, opts)
|
||||
},
|
||||
})
|
||||
|
||||
const replication = local.sync(remote)
|
||||
|
||||
replication.on("complete", () => resolve())
|
||||
|
|
|
@ -24,7 +24,7 @@ exports.getHostingInfo = async () => {
|
|||
_id: HOSTING_DOC,
|
||||
type: exports.HostingTypes.CLOUD,
|
||||
appServerUrl: "app.budi.live",
|
||||
objectStoreUrl: "cdn.app.budi.live",
|
||||
deploymentServerUrl: "",
|
||||
templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
|
||||
useHttps: true,
|
||||
}
|
||||
|
@ -44,6 +44,12 @@ exports.getAppServerUrl = async appId => {
|
|||
return url
|
||||
}
|
||||
|
||||
exports.getDeploymentUrl = async () => {
|
||||
const hostingInfo = await exports.getHostingInfo()
|
||||
const protocol = getProtocol(hostingInfo)
|
||||
return `${protocol}${hostingInfo.deploymentServerUrl}`
|
||||
}
|
||||
|
||||
exports.getTemplatesUrl = async (appId, type, name) => {
|
||||
const hostingInfo = await exports.getHostingInfo()
|
||||
const protocol = getProtocol(hostingInfo)
|
||||
|
|
Loading…
Reference in a new issue