From 4f05bc619b4c2cd8627b965326389bfaee6bf199 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 19 Feb 2024 09:51:10 +0000 Subject: [PATCH 01/31] Handle branding cookies with different tenant ids --- .../builder/src/pages/builder/auth/_layout.svelte | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/auth/_layout.svelte b/packages/builder/src/pages/builder/auth/_layout.svelte index f5484182e8..8902e9ac61 100644 --- a/packages/builder/src/pages/builder/auth/_layout.svelte +++ b/packages/builder/src/pages/builder/auth/_layout.svelte @@ -10,13 +10,18 @@ $redirect("../") } - if ($admin?.checklist?.branding) { + if ($admin.cloud && $admin?.checklist?.branding) { let url = new URL(window.location.href) let hostname = url.hostname let parts = hostname.split(".") - let tenantId = parts[0] + let newTenantId = parts[0] let domain = parts.slice(-2).join(".") - CookieUtils.setCookie("tenantId", tenantId, domain) + + let existingTenantId = CookieUtils.getCookie("tenantId") + + if (!existingTenantId || existingTenantId !== newTenantId) { + CookieUtils.setCookie("tenantId", newTenantId, domain) + } } if ( From c52b6aa0d1e7f836401deebfd16d198d738f4162 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 19 Feb 2024 19:37:18 +0000 Subject: [PATCH 02/31] Some initial test cases for isolated VM. --- .../src/jsRunner/tests/isolatedVM.spec.ts | 33 + .../src/jsRunner/tests/jsRunner.spec.ts | 75 + packages/server/src/jsRunner/tests/marked.txt | 3018 +++++++++++++++++ 3 files changed, 3126 insertions(+) create mode 100644 packages/server/src/jsRunner/tests/isolatedVM.spec.ts create mode 100644 packages/server/src/jsRunner/tests/marked.txt diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts new file mode 100644 index 0000000000..0bd400c8d7 --- /dev/null +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -0,0 +1,33 @@ +import fs from "fs" +import path from "path" +import { IsolatedVM, VM2 } from "../vm" + +function runJSWithIsolatedVM(script: string, context?: any) { + const runner = new IsolatedVM() + if (context) { + runner.withContext(context) + } + return runner.execute(`(function(){\n${script}\n})();`) +} + +function runJSWithVM2(script: string, context?: any) { + const runner = new VM2(context) + return runner.execute(script) +} + +function compare(script: string, context?: any) { + const resultIsolated = runJSWithIsolatedVM(script, context) + const resultVM = runJSWithVM2(script, context) + expect(resultIsolated).toEqual(resultVM) + return resultIsolated +} + +describe("Test isolated vm directly", () => { + it("should handle a very large file", () => { + const marked = fs.readFileSync(path.join(__dirname, "marked.txt"), "utf-8") + const result = compare(marked, { + trigger: { row: { Message: "dddd" } }, + }) + expect(result).toBe("

dddd

\n") + }) +}) diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index 30e29885b1..9bb9052eda 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,5 +1,7 @@ import { validate as isValidUUID } from "uuid" import { processStringSync, encodeJSBinding } from "@budibase/string-templates" +import fs from "fs" +import path from "path" const { runJsHelpersTests } = require("@budibase/string-templates/test/utils") @@ -70,4 +72,77 @@ describe("jsRunner (using isolated-vm)", () => { }) }) }) + + // the test cases here were extracted from templates/real world examples of JS in Budibase + describe("should real world tests of JS", () => { + const context = { + "Unit Value": 2, + Quantity: 1, + } + it("handle test case 1", async () => { + const result = await processJS( + ` + var Gross = $("[Unit Value]") * $("[Quantity]") + return Gross.toFixed(2)`, + context + ) + expect(result).toBeDefined() + expect(result).toBe("2.00") + }) + + it("handle test case 2", async () => { + const context = { + "Purchase Date": "2021-01-21T12:00:00", + } + const result = await processJS( + ` + var purchase = new Date($("[Purchase Date]")); + let purchaseyear = purchase.getFullYear(); + let purchasemonth = purchase.getMonth(); + + var today = new Date (); + let todayyear = today.getFullYear(); + let todaymonth = today.getMonth(); + + var age = todayyear - purchaseyear + + if (((todaymonth - purchasemonth) < 6) == true){ + return age + } + `, + context + ) + expect(result).toBeDefined() + expect(result).toBe(3) + }) + + it("should handle test case 3", async () => { + const context = { + Escalate: true, + "Budget ($)": 1100, + } + const result = await processJS( + ` + if ($("[Escalate]") == true) { + if ($("Budget ($)") <= 1000) + {return 2;} + if ($("Budget ($)") > 1000) + {return 3;} + } + else { + if ($("Budget ($)") <= 1000) + {return 1;} + if ($("Budget ($)") > 1000) + if ($("Budget ($)") < 10000) + {return 2;} + else + {return 3} + } + `, + context + ) + expect(result).toBeDefined() + expect(result).toBe(3) + }) + }) }) diff --git a/packages/server/src/jsRunner/tests/marked.txt b/packages/server/src/jsRunner/tests/marked.txt new file mode 100644 index 0000000000..c5c01d7f0f --- /dev/null +++ b/packages/server/src/jsRunner/tests/marked.txt @@ -0,0 +1,3018 @@ +/** + * marked - a markdown parser + * Copyright (c) 2011-2022, Christopher Jeffrey. (MIT Licensed) + * https://github.com/markedjs/marked + */ + +/** + * DO NOT EDIT THIS FILE + * The code in this file is generated from files in ./src/ + */ + +function getDefaults() { + return { + baseUrl: null, + breaks: false, + extensions: null, + gfm: true, + headerIds: true, + headerPrefix: "", + highlight: null, + langPrefix: "language-", + mangle: true, + pedantic: false, + renderer: null, + sanitize: false, + sanitizer: null, + silent: false, + smartLists: false, + smartypants: false, + tokenizer: null, + walkTokens: null, + xhtml: false, + } +} + +let defaults = getDefaults() + +function changeDefaults(newDefaults) { + defaults = newDefaults +} + +/** + * Helpers + */ +const escapeTest = /[&<>"']/ +const escapeReplace = /[&<>"']/g +const escapeTestNoEncode = /[<>"']|&(?!#?\w+;)/ +const escapeReplaceNoEncode = /[<>"']|&(?!#?\w+;)/g +const escapeReplacements = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", +} +const getEscapeReplacement = ch => escapeReplacements[ch] +function escape(html, encode) { + if (encode) { + if (escapeTest.test(html)) { + return html.replace(escapeReplace, getEscapeReplacement) + } + } else { + if (escapeTestNoEncode.test(html)) { + return html.replace(escapeReplaceNoEncode, getEscapeReplacement) + } + } + + return html +} + +const unescapeTest = /&(#(?:\d+)|(?:#x[0-9A-Fa-f]+)|(?:\w+));?/gi + +/** + * @param {string} html + */ +function unescape(html) { + // explicitly match decimal, hex, and named HTML entities + return html.replace(unescapeTest, (_, n) => { + n = n.toLowerCase() + if (n === "colon") return ":" + if (n.charAt(0) === "#") { + return n.charAt(1) === "x" + ? String.fromCharCode(parseInt(n.substring(2), 16)) + : String.fromCharCode(+n.substring(1)) + } + return "" + }) +} + +const caret = /(^|[^\[])\^/g + +/** + * @param {string | RegExp} regex + * @param {string} opt + */ +function edit(regex, opt) { + regex = typeof regex === "string" ? regex : regex.source + opt = opt || "" + const obj = { + replace: (name, val) => { + val = val.source || val + val = val.replace(caret, "$1") + regex = regex.replace(name, val) + return obj + }, + getRegex: () => { + return new RegExp(regex, opt) + }, + } + return obj +} + +const nonWordAndColonTest = /[^\w:]/g +const originIndependentUrl = /^$|^[a-z][a-z0-9+.-]*:|^[?#]/i + +/** + * @param {boolean} sanitize + * @param {string} base + * @param {string} href + */ +function cleanUrl(sanitize, base, href) { + if (sanitize) { + let prot + try { + prot = decodeURIComponent(unescape(href)) + .replace(nonWordAndColonTest, "") + .toLowerCase() + } catch (e) { + return null + } + if ( + prot.indexOf("javascript:") === 0 || + prot.indexOf("vbscript:") === 0 || + prot.indexOf("data:") === 0 + ) { + return null + } + } + if (base && !originIndependentUrl.test(href)) { + href = resolveUrl(base, href) + } + try { + href = encodeURI(href).replace(/%25/g, "%") + } catch (e) { + return null + } + return href +} + +const baseUrls = {} +const justDomain = /^[^:]+:\/*[^/]*$/ +const protocol = /^([^:]+:)[\s\S]*$/ +const domain = /^([^:]+:\/*[^/]*)[\s\S]*$/ + +/** + * @param {string} base + * @param {string} href + */ +function resolveUrl(base, href) { + if (!baseUrls[" " + base]) { + // we can ignore everything in base after the last slash of its path component, + // but we might need to add _that_ + // https://tools.ietf.org/html/rfc3986#section-3 + if (justDomain.test(base)) { + baseUrls[" " + base] = base + "/" + } else { + baseUrls[" " + base] = rtrim(base, "/", true) + } + } + base = baseUrls[" " + base] + const relativeBase = base.indexOf(":") === -1 + + if (href.substring(0, 2) === "//") { + if (relativeBase) { + return href + } + return base.replace(protocol, "$1") + href + } else if (href.charAt(0) === "/") { + if (relativeBase) { + return href + } + return base.replace(domain, "$1") + href + } else { + return base + href + } +} + +const noopTest = { exec: function noopTest() {} } + +function merge(obj) { + let i = 1, + target, + key + + for (; i < arguments.length; i++) { + target = arguments[i] + for (key in target) { + if (Object.prototype.hasOwnProperty.call(target, key)) { + obj[key] = target[key] + } + } + } + + return obj +} + +function splitCells(tableRow, count) { + // ensure that every cell-delimiting pipe has a space + // before it to distinguish it from an escaped pipe + const row = tableRow.replace(/\|/g, (match, offset, str) => { + let escaped = false, + curr = offset + while (--curr >= 0 && str[curr] === "\\") escaped = !escaped + if (escaped) { + // odd number of slashes means | is escaped + // so we leave it alone + return "|" + } else { + // add space before unescaped | + return " |" + } + }), + cells = row.split(/ \|/) + let i = 0 + + // First/last cell in a row cannot be empty if it has no leading/trailing pipe + if (!cells[0].trim()) { + cells.shift() + } + if (cells.length > 0 && !cells[cells.length - 1].trim()) { + cells.pop() + } + + if (cells.length > count) { + cells.splice(count) + } else { + while (cells.length < count) cells.push("") + } + + for (; i < cells.length; i++) { + // leading or trailing whitespace is ignored per the gfm spec + cells[i] = cells[i].trim().replace(/\\\|/g, "|") + } + return cells +} + +/** + * Remove trailing 'c's. Equivalent to str.replace(/c*$/, ''). + * /c*$/ is vulnerable to REDOS. + * + * @param {string} str + * @param {string} c + * @param {boolean} invert Remove suffix of non-c chars instead. Default falsey. + */ +function rtrim(str, c, invert) { + const l = str.length + if (l === 0) { + return "" + } + + // Length of suffix matching the invert condition. + let suffLen = 0 + + // Step left until we fail to match the invert condition. + while (suffLen < l) { + const currChar = str.charAt(l - suffLen - 1) + if (currChar === c && !invert) { + suffLen++ + } else if (currChar !== c && invert) { + suffLen++ + } else { + break + } + } + + return str.slice(0, l - suffLen) +} + +function findClosingBracket(str, b) { + if (str.indexOf(b[1]) === -1) { + return -1 + } + const l = str.length + let level = 0, + i = 0 + for (; i < l; i++) { + if (str[i] === "\\") { + i++ + } else if (str[i] === b[0]) { + level++ + } else if (str[i] === b[1]) { + level-- + if (level < 0) { + return i + } + } + } + return -1 +} + +function checkSanitizeDeprecation(opt) { + if (opt && opt.sanitize && !opt.silent) { + console.warn( + "marked(): sanitize and sanitizer parameters are deprecated since version 0.7.0, should not be used and will be removed in the future. Read more here: https://marked.js.org/#/USING_ADVANCED.md#options" + ) + } +} + +// copied from https://stackoverflow.com/a/5450113/806777 +/** + * @param {string} pattern + * @param {number} count + */ +function repeatString(pattern, count) { + if (count < 1) { + return "" + } + let result = "" + while (count > 1) { + if (count & 1) { + result += pattern + } + count >>= 1 + pattern += pattern + } + return result + pattern +} + +function outputLink(cap, link, raw, lexer) { + const href = link.href + const title = link.title ? escape(link.title) : null + const text = cap[1].replace(/\\([\[\]])/g, "$1") + + if (cap[0].charAt(0) !== "!") { + lexer.state.inLink = true + const token = { + type: "link", + raw, + href, + title, + text, + tokens: lexer.inlineTokens(text, []), + } + lexer.state.inLink = false + return token + } + return { + type: "image", + raw, + href, + title, + text: escape(text), + } +} + +function indentCodeCompensation(raw, text) { + const matchIndentToCode = raw.match(/^(\s+)(?:```)/) + + if (matchIndentToCode === null) { + return text + } + + const indentToCode = matchIndentToCode[1] + + return text + .split("\n") + .map(node => { + const matchIndentInNode = node.match(/^\s+/) + if (matchIndentInNode === null) { + return node + } + + const [indentInNode] = matchIndentInNode + + if (indentInNode.length >= indentToCode.length) { + return node.slice(indentToCode.length) + } + + return node + }) + .join("\n") +} + +/** + * Tokenizer + */ +class Tokenizer { + constructor(options) { + this.options = options || defaults + } + + space(src) { + const cap = this.rules.block.newline.exec(src) + if (cap && cap[0].length > 0) { + return { + type: "space", + raw: cap[0], + } + } + } + + code(src) { + const cap = this.rules.block.code.exec(src) + if (cap) { + const text = cap[0].replace(/^ {1,4}/gm, "") + return { + type: "code", + raw: cap[0], + codeBlockStyle: "indented", + text: !this.options.pedantic ? rtrim(text, "\n") : text, + } + } + } + + fences(src) { + const cap = this.rules.block.fences.exec(src) + if (cap) { + const raw = cap[0] + const text = indentCodeCompensation(raw, cap[3] || "") + + return { + type: "code", + raw, + lang: cap[2] ? cap[2].trim() : cap[2], + text, + } + } + } + + heading(src) { + const cap = this.rules.block.heading.exec(src) + if (cap) { + let text = cap[2].trim() + + // remove trailing #s + if (/#$/.test(text)) { + const trimmed = rtrim(text, "#") + if (this.options.pedantic) { + text = trimmed.trim() + } else if (!trimmed || / $/.test(trimmed)) { + // CommonMark requires space before trailing #s + text = trimmed.trim() + } + } + + const token = { + type: "heading", + raw: cap[0], + depth: cap[1].length, + text, + tokens: [], + } + this.lexer.inline(token.text, token.tokens) + return token + } + } + + hr(src) { + const cap = this.rules.block.hr.exec(src) + if (cap) { + return { + type: "hr", + raw: cap[0], + } + } + } + + blockquote(src) { + const cap = this.rules.block.blockquote.exec(src) + if (cap) { + const text = cap[0].replace(/^ *>[ \t]?/gm, "") + + return { + type: "blockquote", + raw: cap[0], + tokens: this.lexer.blockTokens(text, []), + text, + } + } + } + + list(src) { + let cap = this.rules.block.list.exec(src) + if (cap) { + let raw, + istask, + ischecked, + indent, + i, + blankLine, + endsWithBlankLine, + line, + nextLine, + rawLine, + itemContents, + endEarly + + let bull = cap[1].trim() + const isordered = bull.length > 1 + + const list = { + type: "list", + raw: "", + ordered: isordered, + start: isordered ? +bull.slice(0, -1) : "", + loose: false, + items: [], + } + + bull = isordered ? `\\d{1,9}\\${bull.slice(-1)}` : `\\${bull}` + + if (this.options.pedantic) { + bull = isordered ? bull : "[*+-]" + } + + // Get next list item + const itemRegex = new RegExp( + `^( {0,3}${bull})((?:[\t ][^\\n]*)?(?:\\n|$))` + ) + + // Check if current bullet point can start a new List Item + while (src) { + endEarly = false + if (!(cap = itemRegex.exec(src))) { + break + } + + if (this.rules.block.hr.test(src)) { + // End list if bullet was actually HR (possibly move into itemRegex?) + break + } + + raw = cap[0] + src = src.substring(raw.length) + + line = cap[2].split("\n", 1)[0] + nextLine = src.split("\n", 1)[0] + + if (this.options.pedantic) { + indent = 2 + itemContents = line.trimLeft() + } else { + indent = cap[2].search(/[^ ]/) // Find first non-space char + indent = indent > 4 ? 1 : indent // Treat indented code blocks (> 4 spaces) as having only 1 indent + itemContents = line.slice(indent) + indent += cap[1].length + } + + blankLine = false + + if (!line && /^ *$/.test(nextLine)) { + // Items begin with at most one blank line + raw += nextLine + "\n" + src = src.substring(nextLine.length + 1) + endEarly = true + } + + if (!endEarly) { + const nextBulletRegex = new RegExp( + `^ {0,${Math.min( + 3, + indent - 1 + )}}(?:[*+-]|\\d{1,9}[.)])((?: [^\\n]*)?(?:\\n|$))` + ) + const hrRegex = new RegExp( + `^ {0,${Math.min( + 3, + indent - 1 + )}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)` + ) + + // Check if following lines should be included in List Item + while (src) { + rawLine = src.split("\n", 1)[0] + line = rawLine + + // Re-align to follow commonmark nesting rules + if (this.options.pedantic) { + line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, " ") + } + + // End list item if found start of new bullet + if (nextBulletRegex.test(line)) { + break + } + + // Horizontal rule found + if (hrRegex.test(src)) { + break + } + + if (line.search(/[^ ]/) >= indent || !line.trim()) { + // Dedent if possible + itemContents += "\n" + line.slice(indent) + } else if (!blankLine) { + // Until blank line, item doesn't need indentation + itemContents += "\n" + line + } else { + // Otherwise, improper indentation ends this item + break + } + + if (!blankLine && !line.trim()) { + // Check if current line is blank + blankLine = true + } + + raw += rawLine + "\n" + src = src.substring(rawLine.length + 1) + } + } + + if (!list.loose) { + // If the previous item ended with a blank line, the list is loose + if (endsWithBlankLine) { + list.loose = true + } else if (/\n *\n *$/.test(raw)) { + endsWithBlankLine = true + } + } + + // Check for task list items + if (this.options.gfm) { + istask = /^\[[ xX]\] /.exec(itemContents) + if (istask) { + ischecked = istask[0] !== "[ ] " + itemContents = itemContents.replace(/^\[[ xX]\] +/, "") + } + } + + list.items.push({ + type: "list_item", + raw, + task: !!istask, + checked: ischecked, + loose: false, + text: itemContents, + }) + + list.raw += raw + } + + // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic + list.items[list.items.length - 1].raw = raw.trimRight() + list.items[list.items.length - 1].text = itemContents.trimRight() + list.raw = list.raw.trimRight() + + const l = list.items.length + + // Item child tokens handled here at end because we needed to have the final item to trim it first + for (i = 0; i < l; i++) { + this.lexer.state.top = false + list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []) + const spacers = list.items[i].tokens.filter(t => t.type === "space") + const hasMultipleLineBreaks = spacers.every(t => { + const chars = t.raw.split("") + let lineBreaks = 0 + for (const char of chars) { + if (char === "\n") { + lineBreaks += 1 + } + if (lineBreaks > 1) { + return true + } + } + + return false + }) + + if (!list.loose && spacers.length && hasMultipleLineBreaks) { + // Having a single line break doesn't mean a list is loose. A single line break is terminating the last list item + list.loose = true + list.items[i].loose = true + } + } + + return list + } + } + + html(src) { + const cap = this.rules.block.html.exec(src) + if (cap) { + const token = { + type: "html", + raw: cap[0], + pre: + !this.options.sanitizer && + (cap[1] === "pre" || cap[1] === "script" || cap[1] === "style"), + text: cap[0], + } + if (this.options.sanitize) { + token.type = "paragraph" + token.text = this.options.sanitizer + ? this.options.sanitizer(cap[0]) + : escape(cap[0]) + token.tokens = [] + this.lexer.inline(token.text, token.tokens) + } + return token + } + } + + def(src) { + const cap = this.rules.block.def.exec(src) + if (cap) { + if (cap[3]) cap[3] = cap[3].substring(1, cap[3].length - 1) + const tag = cap[1].toLowerCase().replace(/\s+/g, " ") + return { + type: "def", + tag, + raw: cap[0], + href: cap[2], + title: cap[3], + } + } + } + + table(src) { + const cap = this.rules.block.table.exec(src) + if (cap) { + const item = { + type: "table", + header: splitCells(cap[1]).map(c => { + return { text: c } + }), + align: cap[2].replace(/^ *|\| *$/g, "").split(/ *\| */), + rows: + cap[3] && cap[3].trim() + ? cap[3].replace(/\n[ \t]*$/, "").split("\n") + : [], + } + + if (item.header.length === item.align.length) { + item.raw = cap[0] + + let l = item.align.length + let i, j, k, row + for (i = 0; i < l; i++) { + if (/^ *-+: *$/.test(item.align[i])) { + item.align[i] = "right" + } else if (/^ *:-+: *$/.test(item.align[i])) { + item.align[i] = "center" + } else if (/^ *:-+ *$/.test(item.align[i])) { + item.align[i] = "left" + } else { + item.align[i] = null + } + } + + l = item.rows.length + for (i = 0; i < l; i++) { + item.rows[i] = splitCells(item.rows[i], item.header.length).map(c => { + return { text: c } + }) + } + + // parse child tokens inside headers and cells + + // header child tokens + l = item.header.length + for (j = 0; j < l; j++) { + item.header[j].tokens = [] + this.lexer.inline(item.header[j].text, item.header[j].tokens) + } + + // cell child tokens + l = item.rows.length + for (j = 0; j < l; j++) { + row = item.rows[j] + for (k = 0; k < row.length; k++) { + row[k].tokens = [] + this.lexer.inline(row[k].text, row[k].tokens) + } + } + + return item + } + } + } + + lheading(src) { + const cap = this.rules.block.lheading.exec(src) + if (cap) { + const token = { + type: "heading", + raw: cap[0], + depth: cap[2].charAt(0) === "=" ? 1 : 2, + text: cap[1], + tokens: [], + } + this.lexer.inline(token.text, token.tokens) + return token + } + } + + paragraph(src) { + const cap = this.rules.block.paragraph.exec(src) + if (cap) { + const token = { + type: "paragraph", + raw: cap[0], + text: + cap[1].charAt(cap[1].length - 1) === "\n" + ? cap[1].slice(0, -1) + : cap[1], + tokens: [], + } + this.lexer.inline(token.text, token.tokens) + return token + } + } + + text(src) { + const cap = this.rules.block.text.exec(src) + if (cap) { + const token = { + type: "text", + raw: cap[0], + text: cap[0], + tokens: [], + } + this.lexer.inline(token.text, token.tokens) + return token + } + } + + escape(src) { + const cap = this.rules.inline.escape.exec(src) + if (cap) { + return { + type: "escape", + raw: cap[0], + text: escape(cap[1]), + } + } + } + + tag(src) { + const cap = this.rules.inline.tag.exec(src) + if (cap) { + if (!this.lexer.state.inLink && /^/i.test(cap[0])) { + this.lexer.state.inLink = false + } + if ( + !this.lexer.state.inRawBlock && + /^<(pre|code|kbd|script)(\s|>)/i.test(cap[0]) + ) { + this.lexer.state.inRawBlock = true + } else if ( + this.lexer.state.inRawBlock && + /^<\/(pre|code|kbd|script)(\s|>)/i.test(cap[0]) + ) { + this.lexer.state.inRawBlock = false + } + + return { + type: this.options.sanitize ? "text" : "html", + raw: cap[0], + inLink: this.lexer.state.inLink, + inRawBlock: this.lexer.state.inRawBlock, + text: this.options.sanitize + ? this.options.sanitizer + ? this.options.sanitizer(cap[0]) + : escape(cap[0]) + : cap[0], + } + } + } + + link(src) { + const cap = this.rules.inline.link.exec(src) + if (cap) { + const trimmedUrl = cap[2].trim() + if (!this.options.pedantic && /^$/.test(trimmedUrl)) { + return + } + + // ending angle bracket cannot be escaped + const rtrimSlash = rtrim(trimmedUrl.slice(0, -1), "\\") + if ((trimmedUrl.length - rtrimSlash.length) % 2 === 0) { + return + } + } else { + // find closing parenthesis + const lastParenIndex = findClosingBracket(cap[2], "()") + if (lastParenIndex > -1) { + const start = cap[0].indexOf("!") === 0 ? 5 : 4 + const linkLen = start + cap[1].length + lastParenIndex + cap[2] = cap[2].substring(0, lastParenIndex) + cap[0] = cap[0].substring(0, linkLen).trim() + cap[3] = "" + } + } + let href = cap[2] + let title = "" + if (this.options.pedantic) { + // split pedantic href and title + const link = /^([^'"]*[^\s])\s+(['"])(.*)\2/.exec(href) + + if (link) { + href = link[1] + title = link[3] + } + } else { + title = cap[3] ? cap[3].slice(1, -1) : "" + } + + href = href.trim() + if (/^$/.test(trimmedUrl)) { + // pedantic allows starting angle bracket without ending angle bracket + href = href.slice(1) + } else { + href = href.slice(1, -1) + } + } + return outputLink( + cap, + { + href: href ? href.replace(this.rules.inline._escapes, "$1") : href, + title: title + ? title.replace(this.rules.inline._escapes, "$1") + : title, + }, + cap[0], + this.lexer + ) + } + } + + reflink(src, links) { + let cap + if ( + (cap = this.rules.inline.reflink.exec(src)) || + (cap = this.rules.inline.nolink.exec(src)) + ) { + let link = (cap[2] || cap[1]).replace(/\s+/g, " ") + link = links[link.toLowerCase()] + if (!link || !link.href) { + const text = cap[0].charAt(0) + return { + type: "text", + raw: text, + text, + } + } + return outputLink(cap, link, cap[0], this.lexer) + } + } + + emStrong(src, maskedSrc, prevChar = "") { + let match = this.rules.inline.emStrong.lDelim.exec(src) + if (!match) return + + // _ can't be between two alphanumerics. \p{L}\p{N} includes non-english alphabet/numbers as well + if (match[3] && prevChar.match(/[\p{L}\p{N}]/u)) return + + const nextChar = match[1] || match[2] || "" + + if ( + !nextChar || + (nextChar && + (prevChar === "" || this.rules.inline.punctuation.exec(prevChar))) + ) { + const lLength = match[0].length - 1 + let rDelim, + rLength, + delimTotal = lLength, + midDelimTotal = 0 + + const endReg = + match[0][0] === "*" + ? this.rules.inline.emStrong.rDelimAst + : this.rules.inline.emStrong.rDelimUnd + endReg.lastIndex = 0 + + // Clip maskedSrc to same section of string as src (move to lexer?) + maskedSrc = maskedSrc.slice(-1 * src.length + lLength) + + while ((match = endReg.exec(maskedSrc)) != null) { + rDelim = + match[1] || match[2] || match[3] || match[4] || match[5] || match[6] + + if (!rDelim) continue // skip single * in __abc*abc__ + + rLength = rDelim.length + + if (match[3] || match[4]) { + // found another Left Delim + delimTotal += rLength + continue + } else if (match[5] || match[6]) { + // either Left or Right Delim + if (lLength % 3 && !((lLength + rLength) % 3)) { + midDelimTotal += rLength + continue // CommonMark Emphasis Rules 9-10 + } + } + + delimTotal -= rLength + + if (delimTotal > 0) continue // Haven't found enough closing delimiters + + // Remove extra characters. *a*** -> *a* + rLength = Math.min(rLength, rLength + delimTotal + midDelimTotal) + + // Create `em` if smallest delimiter has odd char count. *a*** + if (Math.min(lLength, rLength) % 2) { + const text = src.slice(1, lLength + match.index + rLength) + return { + type: "em", + raw: src.slice(0, lLength + match.index + rLength + 1), + text, + tokens: this.lexer.inlineTokens(text, []), + } + } + + // Create 'strong' if smallest delimiter has even char count. **a*** + const text = src.slice(2, lLength + match.index + rLength - 1) + return { + type: "strong", + raw: src.slice(0, lLength + match.index + rLength + 1), + text, + tokens: this.lexer.inlineTokens(text, []), + } + } + } + } + + codespan(src) { + const cap = this.rules.inline.code.exec(src) + if (cap) { + let text = cap[2].replace(/\n/g, " ") + const hasNonSpaceChars = /[^ ]/.test(text) + const hasSpaceCharsOnBothEnds = /^ /.test(text) && / $/.test(text) + if (hasNonSpaceChars && hasSpaceCharsOnBothEnds) { + text = text.substring(1, text.length - 1) + } + text = escape(text, true) + return { + type: "codespan", + raw: cap[0], + text, + } + } + } + + br(src) { + const cap = this.rules.inline.br.exec(src) + if (cap) { + return { + type: "br", + raw: cap[0], + } + } + } + + del(src) { + const cap = this.rules.inline.del.exec(src) + if (cap) { + return { + type: "del", + raw: cap[0], + text: cap[2], + tokens: this.lexer.inlineTokens(cap[2], []), + } + } + } + + autolink(src, mangle) { + const cap = this.rules.inline.autolink.exec(src) + if (cap) { + let text, href + if (cap[2] === "@") { + text = escape(this.options.mangle ? mangle(cap[1]) : cap[1]) + href = "mailto:" + text + } else { + text = escape(cap[1]) + href = text + } + + return { + type: "link", + raw: cap[0], + text, + href, + tokens: [ + { + type: "text", + raw: text, + text, + }, + ], + } + } + } + + url(src, mangle) { + let cap + if ((cap = this.rules.inline.url.exec(src))) { + let text, href + if (cap[2] === "@") { + text = escape(this.options.mangle ? mangle(cap[0]) : cap[0]) + href = "mailto:" + text + } else { + // do extended autolink path validation + let prevCapZero + do { + prevCapZero = cap[0] + cap[0] = this.rules.inline._backpedal.exec(cap[0])[0] + } while (prevCapZero !== cap[0]) + text = escape(cap[0]) + if (cap[1] === "www.") { + href = "http://" + text + } else { + href = text + } + } + return { + type: "link", + raw: cap[0], + text, + href, + tokens: [ + { + type: "text", + raw: text, + text, + }, + ], + } + } + } + + inlineText(src, smartypants) { + const cap = this.rules.inline.text.exec(src) + if (cap) { + let text + if (this.lexer.state.inRawBlock) { + text = this.options.sanitize + ? this.options.sanitizer + ? this.options.sanitizer(cap[0]) + : escape(cap[0]) + : cap[0] + } else { + text = escape(this.options.smartypants ? smartypants(cap[0]) : cap[0]) + } + return { + type: "text", + raw: cap[0], + text, + } + } + } +} + +/** + * Block-Level Grammar + */ +const block = { + newline: /^(?: *(?:\n|$))+/, + code: /^( {4}[^\n]+(?:\n(?: *(?:\n|$))*)?)+/, + fences: + /^ {0,3}(`{3,}(?=[^`\n]*\n)|~{3,})([^\n]*)\n(?:|([\s\S]*?)\n)(?: {0,3}\1[~`]* *(?=\n|$)|$)/, + hr: /^ {0,3}((?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/, + heading: /^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/, + blockquote: /^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/, + list: /^( {0,3}bull)([ \t][^\n]+?)?(?:\n|$)/, + html: + "^ {0,3}(?:" + // optional indentation + "<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:[^\\n]*\\n+|$)" + // (1) + "|comment[^\\n]*(\\n+|$)" + // (2) + "|<\\?[\\s\\S]*?(?:\\?>\\n*|$)" + // (3) + "|\\n*|$)" + // (4) + "|\\n*|$)" + // (5) + "|)[\\s\\S]*?(?:(?:\\n *)+\\n|$)" + // (6) + "|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n *)+\\n|$)" + // (7) open tag + "|(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n *)+\\n|$)" + // (7) closing tag + ")", + def: /^ {0,3}\[(label)\]: *(?:\n *)?]+)>?(?:(?: +(?:\n *)?| *\n *)(title))? *(?:\n+|$)/, + table: noopTest, + lheading: /^([^\n]+)\n {0,3}(=+|-+) *(?:\n+|$)/, + // regex template, placeholders will be replaced according to different paragraph + // interruption rules of commonmark and the original markdown spec: + _paragraph: + /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table| +\n)[^\n]+)*)/, + text: /^[^\n]+/, +} + +block._label = /(?!\s*\])(?:\\.|[^\[\]\\])+/ +block._title = /(?:"(?:\\"?|[^"\\])*"|'[^'\n]*(?:\n[^'\n]+)*\n?'|\([^()]*\))/ +block.def = edit(block.def) + .replace("label", block._label) + .replace("title", block._title) + .getRegex() + +block.bullet = /(?:[*+-]|\d{1,9}[.)])/ +block.listItemStart = edit(/^( *)(bull) */) + .replace("bull", block.bullet) + .getRegex() + +block.list = edit(block.list) + .replace(/bull/g, block.bullet) + .replace( + "hr", + "\\n+(?=\\1?(?:(?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$))" + ) + .replace("def", "\\n+(?=" + block.def.source + ")") + .getRegex() + +block._tag = + "address|article|aside|base|basefont|blockquote|body|caption" + + "|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption" + + "|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe" + + "|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option" + + "|p|param|section|source|summary|table|tbody|td|tfoot|th|thead|title|tr" + + "|track|ul" +block._comment = /|$)/ +block.html = edit(block.html, "i") + .replace("comment", block._comment) + .replace("tag", block._tag) + .replace( + "attribute", + / +[a-zA-Z:_][\w.:-]*(?: *= *"[^"\n]*"| *= *'[^'\n]*'| *= *[^\s"'=<>`]+)?/ + ) + .getRegex() + +block.paragraph = edit(block._paragraph) + .replace("hr", block.hr) + .replace("heading", " {0,3}#{1,6} ") + .replace("|lheading", "") // setex headings don't interrupt commonmark paragraphs + .replace("|table", "") + .replace("blockquote", " {0,3}>") + .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n") + .replace("list", " {0,3}(?:[*+-]|1[.)]) ") // only lists starting from 1 can interrupt + .replace( + "html", + ")|<(?:script|pre|style|textarea|!--)" + ) + .replace("tag", block._tag) // pars can be interrupted by type (6) html blocks + .getRegex() + +block.blockquote = edit(block.blockquote) + .replace("paragraph", block.paragraph) + .getRegex() + +/** + * Normal Block Grammar + */ + +block.normal = merge({}, block) + +/** + * GFM Block Grammar + */ + +block.gfm = merge({}, block.normal, { + table: + "^ *([^\\n ].*\\|.*)\\n" + // Header + " {0,3}(?:\\| *)?(:?-+:? *(?:\\| *:?-+:? *)*)(?:\\| *)?" + // Align + "(?:\\n((?:(?! *\\n|hr|heading|blockquote|code|fences|list|html).*(?:\\n|$))*)\\n*|$)", // Cells +}) + +block.gfm.table = edit(block.gfm.table) + .replace("hr", block.hr) + .replace("heading", " {0,3}#{1,6} ") + .replace("blockquote", " {0,3}>") + .replace("code", " {4}[^\\n]") + .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n") + .replace("list", " {0,3}(?:[*+-]|1[.)]) ") // only lists starting from 1 can interrupt + .replace( + "html", + ")|<(?:script|pre|style|textarea|!--)" + ) + .replace("tag", block._tag) // tables can be interrupted by type (6) html blocks + .getRegex() + +block.gfm.paragraph = edit(block._paragraph) + .replace("hr", block.hr) + .replace("heading", " {0,3}#{1,6} ") + .replace("|lheading", "") // setex headings don't interrupt commonmark paragraphs + .replace("table", block.gfm.table) // interrupt paragraphs with table + .replace("blockquote", " {0,3}>") + .replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n") + .replace("list", " {0,3}(?:[*+-]|1[.)]) ") // only lists starting from 1 can interrupt + .replace( + "html", + ")|<(?:script|pre|style|textarea|!--)" + ) + .replace("tag", block._tag) // pars can be interrupted by type (6) html blocks + .getRegex() +/** + * Pedantic grammar (original John Gruber's loose markdown specification) + */ + +block.pedantic = merge({}, block.normal, { + html: edit( + "^ *(?:comment *(?:\\n|\\s*$)" + + "|<(tag)[\\s\\S]+? *(?:\\n{2,}|\\s*$)" + // closed tag + "|\\s]*)*?/?> *(?:\\n{2,}|\\s*$))" + ) + .replace("comment", block._comment) + .replace( + /tag/g, + "(?!(?:" + + "a|em|strong|small|s|cite|q|dfn|abbr|data|time|code|var|samp|kbd|sub" + + "|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo|span|br|wbr|ins|del|img)" + + "\\b)\\w+(?!:|[^\\w\\s@]*@)\\b" + ) + .getRegex(), + def: /^ *\[([^\]]+)\]: *]+)>?(?: +(["(][^\n]+[")]))? *(?:\n+|$)/, + heading: /^(#{1,6})(.*)(?:\n+|$)/, + fences: noopTest, // fences not supported + paragraph: edit(block.normal._paragraph) + .replace("hr", block.hr) + .replace("heading", " *#{1,6} *[^\n]") + .replace("lheading", block.lheading) + .replace("blockquote", " {0,3}>") + .replace("|fences", "") + .replace("|list", "") + .replace("|html", "") + .getRegex(), +}) + +/** + * Inline-Level Grammar + */ +const inline = { + escape: /^\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/, + autolink: /^<(scheme:[^\s\x00-\x1f<>]*|email)>/, + url: noopTest, + tag: + "^comment" + + "|^" + // self-closing tag + "|^<[a-zA-Z][\\w-]*(?:attribute)*?\\s*/?>" + // open tag + "|^<\\?[\\s\\S]*?\\?>" + // processing instruction, e.g. + "|^" + // declaration, e.g. + "|^", // CDATA section + link: /^!?\[(label)\]\(\s*(href)(?:\s+(title))?\s*\)/, + reflink: /^!?\[(label)\]\[(ref)\]/, + nolink: /^!?\[(ref)\](?:\[\])?/, + reflinkSearch: "reflink|nolink(?!\\()", + emStrong: { + lDelim: /^(?:\*+(?:([punct_])|[^\s*]))|^_+(?:([punct*])|([^\s_]))/, + // (1) and (2) can only be a Right Delimiter. (3) and (4) can only be Left. (5) and (6) can be either Left or Right. + // () Skip orphan inside strong () Consume to delim (1) #*** (2) a***#, a*** (3) #***a, ***a (4) ***# (5) #***# (6) a***a + rDelimAst: + /^[^_*]*?\_\_[^_*]*?\*[^_*]*?(?=\_\_)|[^*]+(?=[^*])|[punct_](\*+)(?=[\s]|$)|[^punct*_\s](\*+)(?=[punct_\s]|$)|[punct_\s](\*+)(?=[^punct*_\s])|[\s](\*+)(?=[punct_])|[punct_](\*+)(?=[punct_])|[^punct*_\s](\*+)(?=[^punct*_\s])/, + rDelimUnd: + /^[^_*]*?\*\*[^_*]*?\_[^_*]*?(?=\*\*)|[^_]+(?=[^_])|[punct*](\_+)(?=[\s]|$)|[^punct*_\s](\_+)(?=[punct*\s]|$)|[punct*\s](\_+)(?=[^punct*_\s])|[\s](\_+)(?=[punct*])|[punct*](\_+)(?=[punct*])/, // ^- Not allowed for _ + }, + code: /^(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/, + br: /^( {2,}|\\)\n(?!\s*$)/, + del: noopTest, + text: /^(`+|[^`])(?:(?= {2,}\n)|[\s\S]*?(?:(?=[\\?@\\[\\]`^{|}~" +inline.punctuation = edit(inline.punctuation) + .replace(/punctuation/g, inline._punctuation) + .getRegex() + +// sequences em should skip over [title](link), `code`, +inline.blockSkip = /\[[^\]]*?\]\([^\)]*?\)|`[^`]*?`|<[^>]*?>/g +inline.escapedEmSt = /\\\*|\\_/g + +inline._comment = edit(block._comment).replace("(?:-->|$)", "-->").getRegex() + +inline.emStrong.lDelim = edit(inline.emStrong.lDelim) + .replace(/punct/g, inline._punctuation) + .getRegex() + +inline.emStrong.rDelimAst = edit(inline.emStrong.rDelimAst, "g") + .replace(/punct/g, inline._punctuation) + .getRegex() + +inline.emStrong.rDelimUnd = edit(inline.emStrong.rDelimUnd, "g") + .replace(/punct/g, inline._punctuation) + .getRegex() + +inline._escapes = /\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/g + +inline._scheme = /[a-zA-Z][a-zA-Z0-9+.-]{1,31}/ +inline._email = + /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+(@)[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(?![-_])/ +inline.autolink = edit(inline.autolink) + .replace("scheme", inline._scheme) + .replace("email", inline._email) + .getRegex() + +inline._attribute = + /\s+[a-zA-Z:_][\w.:-]*(?:\s*=\s*"[^"]*"|\s*=\s*'[^']*'|\s*=\s*[^\s"'=<>`]+)?/ + +inline.tag = edit(inline.tag) + .replace("comment", inline._comment) + .replace("attribute", inline._attribute) + .getRegex() + +inline._label = /(?:\[(?:\\.|[^\[\]\\])*\]|\\.|`[^`]*`|[^\[\]\\`])*?/ +inline._href = /<(?:\\.|[^\n<>\\])+>|[^\s\x00-\x1f]*/ +inline._title = /"(?:\\"?|[^"\\])*"|'(?:\\'?|[^'\\])*'|\((?:\\\)?|[^)\\])*\)/ + +inline.link = edit(inline.link) + .replace("label", inline._label) + .replace("href", inline._href) + .replace("title", inline._title) + .getRegex() + +inline.reflink = edit(inline.reflink) + .replace("label", inline._label) + .replace("ref", block._label) + .getRegex() + +inline.nolink = edit(inline.nolink).replace("ref", block._label).getRegex() + +inline.reflinkSearch = edit(inline.reflinkSearch, "g") + .replace("reflink", inline.reflink) + .replace("nolink", inline.nolink) + .getRegex() + +/** + * Normal Inline Grammar + */ + +inline.normal = merge({}, inline) + +/** + * Pedantic Inline Grammar + */ + +inline.pedantic = merge({}, inline.normal, { + strong: { + start: /^__|\*\*/, + middle: /^__(?=\S)([\s\S]*?\S)__(?!_)|^\*\*(?=\S)([\s\S]*?\S)\*\*(?!\*)/, + endAst: /\*\*(?!\*)/g, + endUnd: /__(?!_)/g, + }, + em: { + start: /^_|\*/, + middle: /^()\*(?=\S)([\s\S]*?\S)\*(?!\*)|^_(?=\S)([\s\S]*?\S)_(?!_)/, + endAst: /\*(?!\*)/g, + endUnd: /_(?!_)/g, + }, + link: edit(/^!?\[(label)\]\((.*?)\)/) + .replace("label", inline._label) + .getRegex(), + reflink: edit(/^!?\[(label)\]\s*\[([^\]]*)\]/) + .replace("label", inline._label) + .getRegex(), +}) + +/** + * GFM Inline Grammar + */ + +inline.gfm = merge({}, inline.normal, { + escape: edit(inline.escape).replace("])", "~|])").getRegex(), + _extended_email: + /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/, + url: /^((?:ftp|https?):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.?)+[^\s<]*|^email/, + _backpedal: + /(?:[^?!.,:;*_~()&]+|\([^)]*\)|&(?![a-zA-Z0-9]+;$)|[?!.,:;*_~)]+(?!$))+/, + del: /^(~~?)(?=[^\s~])([\s\S]*?[^\s~])\1(?=[^~]|$)/, + text: /^([`~]+|[^`~])(?:(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\ 0.5) { + ch = "x" + ch.toString(16) + } + out += "&#" + ch + ";" + } + + return out +} + +/** + * Block Lexer + */ +class Lexer { + constructor(options) { + this.tokens = [] + this.tokens.links = Object.create(null) + this.options = options || defaults + this.options.tokenizer = this.options.tokenizer || new Tokenizer() + this.tokenizer = this.options.tokenizer + this.tokenizer.options = this.options + this.tokenizer.lexer = this + this.inlineQueue = [] + this.state = { + inLink: false, + inRawBlock: false, + top: true, + } + + const rules = { + block: block.normal, + inline: inline.normal, + } + + if (this.options.pedantic) { + rules.block = block.pedantic + rules.inline = inline.pedantic + } else if (this.options.gfm) { + rules.block = block.gfm + if (this.options.breaks) { + rules.inline = inline.breaks + } else { + rules.inline = inline.gfm + } + } + this.tokenizer.rules = rules + } + + /** + * Expose Rules + */ + static get rules() { + return { + block, + inline, + } + } + + /** + * Static Lex Method + */ + static lex(src, options) { + const lexer = new Lexer(options) + return lexer.lex(src) + } + + /** + * Static Lex Inline Method + */ + static lexInline(src, options) { + const lexer = new Lexer(options) + return lexer.inlineTokens(src) + } + + /** + * Preprocessing + */ + lex(src) { + src = src.replace(/\r\n|\r/g, "\n") + + this.blockTokens(src, this.tokens) + + let next + while ((next = this.inlineQueue.shift())) { + this.inlineTokens(next.src, next.tokens) + } + + return this.tokens + } + + /** + * Lexing + */ + blockTokens(src, tokens = []) { + if (this.options.pedantic) { + src = src.replace(/\t/g, " ").replace(/^ +$/gm, "") + } else { + src = src.replace(/^( *)(\t+)/gm, (_, leading, tabs) => { + return leading + " ".repeat(tabs.length) + }) + } + + let token, lastToken, cutSrc, lastParagraphClipped + + while (src) { + if ( + this.options.extensions && + this.options.extensions.block && + this.options.extensions.block.some(extTokenizer => { + if ((token = extTokenizer.call({ lexer: this }, src, tokens))) { + src = src.substring(token.raw.length) + tokens.push(token) + return true + } + return false + }) + ) { + continue + } + + // newline + if ((token = this.tokenizer.space(src))) { + src = src.substring(token.raw.length) + if (token.raw.length === 1 && tokens.length > 0) { + // if there's a single \n as a spacer, it's terminating the last line, + // so move it there so that we don't get unecessary paragraph tags + tokens[tokens.length - 1].raw += "\n" + } else { + tokens.push(token) + } + continue + } + + // code + if ((token = this.tokenizer.code(src))) { + src = src.substring(token.raw.length) + lastToken = tokens[tokens.length - 1] + // An indented code block cannot interrupt a paragraph. + if ( + lastToken && + (lastToken.type === "paragraph" || lastToken.type === "text") + ) { + lastToken.raw += "\n" + token.raw + lastToken.text += "\n" + token.text + this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text + } else { + tokens.push(token) + } + continue + } + + // fences + if ((token = this.tokenizer.fences(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // heading + if ((token = this.tokenizer.heading(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // hr + if ((token = this.tokenizer.hr(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // blockquote + if ((token = this.tokenizer.blockquote(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // list + if ((token = this.tokenizer.list(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // html + if ((token = this.tokenizer.html(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // def + if ((token = this.tokenizer.def(src))) { + src = src.substring(token.raw.length) + lastToken = tokens[tokens.length - 1] + if ( + lastToken && + (lastToken.type === "paragraph" || lastToken.type === "text") + ) { + lastToken.raw += "\n" + token.raw + lastToken.text += "\n" + token.raw + this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text + } else if (!this.tokens.links[token.tag]) { + this.tokens.links[token.tag] = { + href: token.href, + title: token.title, + } + } + continue + } + + // table (gfm) + if ((token = this.tokenizer.table(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // lheading + if ((token = this.tokenizer.lheading(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // top-level paragraph + // prevent paragraph consuming extensions by clipping 'src' to extension start + cutSrc = src + if (this.options.extensions && this.options.extensions.startBlock) { + let startIndex = Infinity + const tempSrc = src.slice(1) + let tempStart + this.options.extensions.startBlock.forEach(function (getStartIndex) { + tempStart = getStartIndex.call({ lexer: this }, tempSrc) + if (typeof tempStart === "number" && tempStart >= 0) { + startIndex = Math.min(startIndex, tempStart) + } + }) + if (startIndex < Infinity && startIndex >= 0) { + cutSrc = src.substring(0, startIndex + 1) + } + } + if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) { + lastToken = tokens[tokens.length - 1] + if (lastParagraphClipped && lastToken.type === "paragraph") { + lastToken.raw += "\n" + token.raw + lastToken.text += "\n" + token.text + this.inlineQueue.pop() + this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text + } else { + tokens.push(token) + } + lastParagraphClipped = cutSrc.length !== src.length + src = src.substring(token.raw.length) + continue + } + + // text + if ((token = this.tokenizer.text(src))) { + src = src.substring(token.raw.length) + lastToken = tokens[tokens.length - 1] + if (lastToken && lastToken.type === "text") { + lastToken.raw += "\n" + token.raw + lastToken.text += "\n" + token.text + this.inlineQueue.pop() + this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text + } else { + tokens.push(token) + } + continue + } + + if (src) { + const errMsg = "Infinite loop on byte: " + src.charCodeAt(0) + if (this.options.silent) { + console.error(errMsg) + break + } else { + throw new Error(errMsg) + } + } + } + + this.state.top = true + return tokens + } + + inline(src, tokens) { + this.inlineQueue.push({ src, tokens }) + } + + /** + * Lexing/Compiling + */ + inlineTokens(src, tokens = []) { + let token, lastToken, cutSrc + + // String with links masked to avoid interference with em and strong + let maskedSrc = src + let match + let keepPrevChar, prevChar + + // Mask out reflinks + if (this.tokens.links) { + const links = Object.keys(this.tokens.links) + if (links.length > 0) { + while ( + (match = this.tokenizer.rules.inline.reflinkSearch.exec(maskedSrc)) != + null + ) { + if ( + links.includes(match[0].slice(match[0].lastIndexOf("[") + 1, -1)) + ) { + maskedSrc = + maskedSrc.slice(0, match.index) + + "[" + + repeatString("a", match[0].length - 2) + + "]" + + maskedSrc.slice( + this.tokenizer.rules.inline.reflinkSearch.lastIndex + ) + } + } + } + } + // Mask out other blocks + while ( + (match = this.tokenizer.rules.inline.blockSkip.exec(maskedSrc)) != null + ) { + maskedSrc = + maskedSrc.slice(0, match.index) + + "[" + + repeatString("a", match[0].length - 2) + + "]" + + maskedSrc.slice(this.tokenizer.rules.inline.blockSkip.lastIndex) + } + + // Mask out escaped em & strong delimiters + while ( + (match = this.tokenizer.rules.inline.escapedEmSt.exec(maskedSrc)) != null + ) { + maskedSrc = + maskedSrc.slice(0, match.index) + + "++" + + maskedSrc.slice(this.tokenizer.rules.inline.escapedEmSt.lastIndex) + } + + while (src) { + if (!keepPrevChar) { + prevChar = "" + } + keepPrevChar = false + + // extensions + if ( + this.options.extensions && + this.options.extensions.inline && + this.options.extensions.inline.some(extTokenizer => { + if ((token = extTokenizer.call({ lexer: this }, src, tokens))) { + src = src.substring(token.raw.length) + tokens.push(token) + return true + } + return false + }) + ) { + continue + } + + // escape + if ((token = this.tokenizer.escape(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // tag + if ((token = this.tokenizer.tag(src))) { + src = src.substring(token.raw.length) + lastToken = tokens[tokens.length - 1] + if (lastToken && token.type === "text" && lastToken.type === "text") { + lastToken.raw += token.raw + lastToken.text += token.text + } else { + tokens.push(token) + } + continue + } + + // link + if ((token = this.tokenizer.link(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // reflink, nolink + if ((token = this.tokenizer.reflink(src, this.tokens.links))) { + src = src.substring(token.raw.length) + lastToken = tokens[tokens.length - 1] + if (lastToken && token.type === "text" && lastToken.type === "text") { + lastToken.raw += token.raw + lastToken.text += token.text + } else { + tokens.push(token) + } + continue + } + + // em & strong + if ((token = this.tokenizer.emStrong(src, maskedSrc, prevChar))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // code + if ((token = this.tokenizer.codespan(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // br + if ((token = this.tokenizer.br(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // del (gfm) + if ((token = this.tokenizer.del(src))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // autolink + if ((token = this.tokenizer.autolink(src, mangle))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // url (gfm) + if (!this.state.inLink && (token = this.tokenizer.url(src, mangle))) { + src = src.substring(token.raw.length) + tokens.push(token) + continue + } + + // text + // prevent inlineText consuming extensions by clipping 'src' to extension start + cutSrc = src + if (this.options.extensions && this.options.extensions.startInline) { + let startIndex = Infinity + const tempSrc = src.slice(1) + let tempStart + this.options.extensions.startInline.forEach(function (getStartIndex) { + tempStart = getStartIndex.call({ lexer: this }, tempSrc) + if (typeof tempStart === "number" && tempStart >= 0) { + startIndex = Math.min(startIndex, tempStart) + } + }) + if (startIndex < Infinity && startIndex >= 0) { + cutSrc = src.substring(0, startIndex + 1) + } + } + if ((token = this.tokenizer.inlineText(cutSrc, smartypants))) { + src = src.substring(token.raw.length) + if (token.raw.slice(-1) !== "_") { + // Track prevChar before string of ____ started + prevChar = token.raw.slice(-1) + } + keepPrevChar = true + lastToken = tokens[tokens.length - 1] + if (lastToken && lastToken.type === "text") { + lastToken.raw += token.raw + lastToken.text += token.text + } else { + tokens.push(token) + } + continue + } + + if (src) { + const errMsg = "Infinite loop on byte: " + src.charCodeAt(0) + if (this.options.silent) { + console.error(errMsg) + break + } else { + throw new Error(errMsg) + } + } + } + + return tokens + } +} + +/** + * Renderer + */ +class Renderer { + constructor(options) { + this.options = options || defaults + } + + code(code, infostring, escaped) { + const lang = (infostring || "").match(/\S*/)[0] + if (this.options.highlight) { + const out = this.options.highlight(code, lang) + if (out != null && out !== code) { + escaped = true + code = out + } + } + + code = code.replace(/\n$/, "") + "\n" + + if (!lang) { + return ( + "
" +
+        (escaped ? code : escape(code, true)) +
+        "
\n" + ) + } + + return ( + '
' +
+      (escaped ? code : escape(code, true)) +
+      "
\n" + ) + } + + /** + * @param {string} quote + */ + blockquote(quote) { + return `
\n${quote}
\n` + } + + html(html) { + return html + } + + /** + * @param {string} text + * @param {string} level + * @param {string} raw + * @param {any} slugger + */ + heading(text, level, raw, slugger) { + if (this.options.headerIds) { + const id = this.options.headerPrefix + slugger.slug(raw) + return `${text}\n` + } + + // ignore IDs + return `${text}\n` + } + + hr() { + return this.options.xhtml ? "
\n" : "
\n" + } + + list(body, ordered, start) { + const type = ordered ? "ol" : "ul", + startatt = ordered && start !== 1 ? ' start="' + start + '"' : "" + return "<" + type + startatt + ">\n" + body + "\n" + } + + /** + * @param {string} text + */ + listitem(text) { + return `
  • ${text}
    • \n` + } + + checkbox(checked) { + return ( + " " + ) + } + + /** + * @param {string} text + */ + paragraph(text) { + return `

    ${text}

    \n` + } + + /** + * @param {string} header + * @param {string} body + */ + table(header, body) { + if (body) body = `${body}` + + return ( + "\n" + "\n" + header + "\n" + body + "
    \n" + ) + } + + /** + * @param {string} content + */ + tablerow(content) { + return `\n${content}\n` + } + + tablecell(content, flags) { + const type = flags.header ? "th" : "td" + const tag = flags.align ? `<${type} align="${flags.align}">` : `<${type}>` + return tag + content + `\n` + } + + /** + * span level renderer + * @param {string} text + */ + strong(text) { + return `${text}` + } + + /** + * @param {string} text + */ + em(text) { + return `${text}` + } + + /** + * @param {string} text + */ + codespan(text) { + return `${text}` + } + + br() { + return this.options.xhtml ? "
    " : "
    " + } + + /** + * @param {string} text + */ + del(text) { + return `${text}` + } + + /** + * @param {string} href + * @param {string} title + * @param {string} text + */ + link(href, title, text) { + href = cleanUrl(this.options.sanitize, this.options.baseUrl, href) + if (href === null) { + return text + } + let out = '    " + return out + } + + /** + * @param {string} href + * @param {string} title + * @param {string} text + */ + image(href, title, text) { + href = cleanUrl(this.options.sanitize, this.options.baseUrl, href) + if (href === null) { + return text + } + + let out = `${text}" : ">" + return out + } + + text(text) { + return text + } +} + +/** + * TextRenderer + * returns only the textual part of the token + */ +class TextRenderer { + // no need for block level renderers + strong(text) { + return text + } + + em(text) { + return text + } + + codespan(text) { + return text + } + + del(text) { + return text + } + + html(text) { + return text + } + + text(text) { + return text + } + + link(href, title, text) { + return "" + text + } + + image(href, title, text) { + return "" + text + } + + br() { + return "" + } +} + +/** + * Slugger generates header id + */ +class Slugger { + constructor() { + this.seen = {} + } + + /** + * @param {string} value + */ + serialize(value) { + return ( + value + .toLowerCase() + .trim() + // remove html tags + .replace(/<[!\/a-z].*?>/gi, "") + // remove unwanted chars + .replace( + /[\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,./:;<=>?@[\]^`{|}~]/g, + "" + ) + .replace(/\s/g, "-") + ) + } + + /** + * Finds the next safe (unique) slug to use + * @param {string} originalSlug + * @param {boolean} isDryRun + */ + getNextSafeSlug(originalSlug, isDryRun) { + let slug = originalSlug + let occurenceAccumulator = 0 + if (this.seen.hasOwnProperty(slug)) { + occurenceAccumulator = this.seen[originalSlug] + do { + occurenceAccumulator++ + slug = originalSlug + "-" + occurenceAccumulator + } while (this.seen.hasOwnProperty(slug)) + } + if (!isDryRun) { + this.seen[originalSlug] = occurenceAccumulator + this.seen[slug] = 0 + } + return slug + } + + /** + * Convert string to unique id + * @param {object} [options] + * @param {boolean} [options.dryrun] Generates the next unique slug without + * updating the internal accumulator. + */ + slug(value, options = {}) { + const slug = this.serialize(value) + return this.getNextSafeSlug(slug, options.dryrun) + } +} + +/** + * Parsing & Compiling + */ +class Parser { + constructor(options) { + this.options = options || defaults + this.options.renderer = this.options.renderer || new Renderer() + this.renderer = this.options.renderer + this.renderer.options = this.options + this.textRenderer = new TextRenderer() + this.slugger = new Slugger() + } + + /** + * Static Parse Method + */ + static parse(tokens, options) { + const parser = new Parser(options) + return parser.parse(tokens) + } + + /** + * Static Parse Inline Method + */ + static parseInline(tokens, options) { + const parser = new Parser(options) + return parser.parseInline(tokens) + } + + /** + * Parse Loop + */ + parse(tokens, top = true) { + let out = "", + i, + j, + k, + l2, + l3, + row, + cell, + header, + body, + token, + ordered, + start, + loose, + itemBody, + item, + checked, + task, + checkbox, + ret + + const l = tokens.length + for (i = 0; i < l; i++) { + token = tokens[i] + + // Run any renderer extensions + if ( + this.options.extensions && + this.options.extensions.renderers && + this.options.extensions.renderers[token.type] + ) { + ret = this.options.extensions.renderers[token.type].call( + { parser: this }, + token + ) + if ( + ret !== false || + ![ + "space", + "hr", + "heading", + "code", + "table", + "blockquote", + "list", + "html", + "paragraph", + "text", + ].includes(token.type) + ) { + out += ret || "" + continue + } + } + + switch (token.type) { + case "space": { + continue + } + case "hr": { + out += this.renderer.hr() + continue + } + case "heading": { + out += this.renderer.heading( + this.parseInline(token.tokens), + token.depth, + unescape(this.parseInline(token.tokens, this.textRenderer)), + this.slugger + ) + continue + } + case "code": { + out += this.renderer.code(token.text, token.lang, token.escaped) + continue + } + case "table": { + header = "" + + // header + cell = "" + l2 = token.header.length + for (j = 0; j < l2; j++) { + cell += this.renderer.tablecell( + this.parseInline(token.header[j].tokens), + { header: true, align: token.align[j] } + ) + } + header += this.renderer.tablerow(cell) + + body = "" + l2 = token.rows.length + for (j = 0; j < l2; j++) { + row = token.rows[j] + + cell = "" + l3 = row.length + for (k = 0; k < l3; k++) { + cell += this.renderer.tablecell(this.parseInline(row[k].tokens), { + header: false, + align: token.align[k], + }) + } + + body += this.renderer.tablerow(cell) + } + out += this.renderer.table(header, body) + continue + } + case "blockquote": { + body = this.parse(token.tokens) + out += this.renderer.blockquote(body) + continue + } + case "list": { + ordered = token.ordered + start = token.start + loose = token.loose + l2 = token.items.length + + body = "" + for (j = 0; j < l2; j++) { + item = token.items[j] + checked = item.checked + task = item.task + + itemBody = "" + if (item.task) { + checkbox = this.renderer.checkbox(checked) + if (loose) { + if ( + item.tokens.length > 0 && + item.tokens[0].type === "paragraph" + ) { + item.tokens[0].text = checkbox + " " + item.tokens[0].text + if ( + item.tokens[0].tokens && + item.tokens[0].tokens.length > 0 && + item.tokens[0].tokens[0].type === "text" + ) { + item.tokens[0].tokens[0].text = + checkbox + " " + item.tokens[0].tokens[0].text + } + } else { + item.tokens.unshift({ + type: "text", + text: checkbox, + }) + } + } else { + itemBody += checkbox + } + } + + itemBody += this.parse(item.tokens, loose) + body += this.renderer.listitem(itemBody, task, checked) + } + + out += this.renderer.list(body, ordered, start) + continue + } + case "html": { + // TODO parse inline content if parameter markdown=1 + out += this.renderer.html(token.text) + continue + } + case "paragraph": { + out += this.renderer.paragraph(this.parseInline(token.tokens)) + continue + } + case "text": { + body = token.tokens ? this.parseInline(token.tokens) : token.text + while (i + 1 < l && tokens[i + 1].type === "text") { + token = tokens[++i] + body += + "\n" + + (token.tokens ? this.parseInline(token.tokens) : token.text) + } + out += top ? this.renderer.paragraph(body) : body + continue + } + + default: { + const errMsg = 'Token with "' + token.type + '" type was not found.' + if (this.options.silent) { + console.error(errMsg) + return + } else { + throw new Error(errMsg) + } + } + } + } + + return out + } + + /** + * Parse Inline Tokens + */ + parseInline(tokens, renderer) { + renderer = renderer || this.renderer + let out = "", + i, + token, + ret + + const l = tokens.length + for (i = 0; i < l; i++) { + token = tokens[i] + + // Run any renderer extensions + if ( + this.options.extensions && + this.options.extensions.renderers && + this.options.extensions.renderers[token.type] + ) { + ret = this.options.extensions.renderers[token.type].call( + { parser: this }, + token + ) + if ( + ret !== false || + ![ + "escape", + "html", + "link", + "image", + "strong", + "em", + "codespan", + "br", + "del", + "text", + ].includes(token.type) + ) { + out += ret || "" + continue + } + } + + switch (token.type) { + case "escape": { + out += renderer.text(token.text) + break + } + case "html": { + out += renderer.html(token.text) + break + } + case "link": { + out += renderer.link( + token.href, + token.title, + this.parseInline(token.tokens, renderer) + ) + break + } + case "image": { + out += renderer.image(token.href, token.title, token.text) + break + } + case "strong": { + out += renderer.strong(this.parseInline(token.tokens, renderer)) + break + } + case "em": { + out += renderer.em(this.parseInline(token.tokens, renderer)) + break + } + case "codespan": { + out += renderer.codespan(token.text) + break + } + case "br": { + out += renderer.br() + break + } + case "del": { + out += renderer.del(this.parseInline(token.tokens, renderer)) + break + } + case "text": { + out += renderer.text(token.text) + break + } + default: { + const errMsg = 'Token with "' + token.type + '" type was not found.' + if (this.options.silent) { + console.error(errMsg) + return + } else { + throw new Error(errMsg) + } + } + } + } + return out + } +} + +/** + * Marked + */ +function marked(src, opt, callback) { + // throw error in case of non string input + if (typeof src === "undefined" || src === null) { + throw new Error("marked(): input parameter is undefined or null") + } + if (typeof src !== "string") { + throw new Error( + "marked(): input parameter is of type " + + Object.prototype.toString.call(src) + + ", string expected" + ) + } + + if (typeof opt === "function") { + callback = opt + opt = null + } + + opt = merge({}, marked.defaults, opt || {}) + checkSanitizeDeprecation(opt) + + if (callback) { + const highlight = opt.highlight + let tokens + + try { + tokens = Lexer.lex(src, opt) + } catch (e) { + return callback(e) + } + + const done = function (err) { + let out + + if (!err) { + try { + if (opt.walkTokens) { + marked.walkTokens(tokens, opt.walkTokens) + } + out = Parser.parse(tokens, opt) + } catch (e) { + err = e + } + } + + opt.highlight = highlight + + return err ? callback(err) : callback(null, out) + } + + if (!highlight || highlight.length < 3) { + return done() + } + + delete opt.highlight + + if (!tokens.length) return done() + + let pending = 0 + marked.walkTokens(tokens, function (token) { + if (token.type === "code") { + pending++ + setTimeout(() => { + highlight(token.text, token.lang, function (err, code) { + if (err) { + return done(err) + } + if (code != null && code !== token.text) { + token.text = code + token.escaped = true + } + + pending-- + if (pending === 0) { + done() + } + }) + }, 0) + } + }) + + if (pending === 0) { + done() + } + + return + } + + try { + const tokens = Lexer.lex(src, opt) + if (opt.walkTokens) { + marked.walkTokens(tokens, opt.walkTokens) + } + return Parser.parse(tokens, opt) + } catch (e) { + e.message += "\nPlease report this to https://github.com/markedjs/marked." + if (opt.silent) { + return ( + "

    An error occurred:

    " +
+        escape(e.message + "", true) +
+        "
    " + ) + } + throw e + } +} + +/** + * Options + */ + +marked.options = marked.setOptions = function (opt) { + merge(marked.defaults, opt) + changeDefaults(marked.defaults) + return marked +} + +marked.getDefaults = getDefaults + +marked.defaults = defaults + +/** + * Use Extension + */ + +marked.use = function (...args) { + const opts = merge({}, ...args) + const extensions = marked.defaults.extensions || { + renderers: {}, + childTokens: {}, + } + let hasExtensions + + args.forEach(pack => { + // ==-- Parse "addon" extensions --== // + if (pack.extensions) { + hasExtensions = true + pack.extensions.forEach(ext => { + if (!ext.name) { + throw new Error("extension name required") + } + if (ext.renderer) { + // Renderer extensions + const prevRenderer = extensions.renderers + ? extensions.renderers[ext.name] + : null + if (prevRenderer) { + // Replace extension with func to run new extension but fall back if false + extensions.renderers[ext.name] = function (...args) { + let ret = ext.renderer.apply(this, args) + if (ret === false) { + ret = prevRenderer.apply(this, args) + } + return ret + } + } else { + extensions.renderers[ext.name] = ext.renderer + } + } + if (ext.tokenizer) { + // Tokenizer Extensions + if (!ext.level || (ext.level !== "block" && ext.level !== "inline")) { + throw new Error("extension level must be 'block' or 'inline'") + } + if (extensions[ext.level]) { + extensions[ext.level].unshift(ext.tokenizer) + } else { + extensions[ext.level] = [ext.tokenizer] + } + if (ext.start) { + // Function to check for start of token + if (ext.level === "block") { + if (extensions.startBlock) { + extensions.startBlock.push(ext.start) + } else { + extensions.startBlock = [ext.start] + } + } else if (ext.level === "inline") { + if (extensions.startInline) { + extensions.startInline.push(ext.start) + } else { + extensions.startInline = [ext.start] + } + } + } + } + if (ext.childTokens) { + // Child tokens to be visited by walkTokens + extensions.childTokens[ext.name] = ext.childTokens + } + }) + } + + // ==-- Parse "overwrite" extensions --== // + if (pack.renderer) { + const renderer = marked.defaults.renderer || new Renderer() + for (const prop in pack.renderer) { + const prevRenderer = renderer[prop] + // Replace renderer with func to run extension, but fall back if false + renderer[prop] = (...args) => { + let ret = pack.renderer[prop].apply(renderer, args) + if (ret === false) { + ret = prevRenderer.apply(renderer, args) + } + return ret + } + } + opts.renderer = renderer + } + if (pack.tokenizer) { + const tokenizer = marked.defaults.tokenizer || new Tokenizer() + for (const prop in pack.tokenizer) { + const prevTokenizer = tokenizer[prop] + // Replace tokenizer with func to run extension, but fall back if false + tokenizer[prop] = (...args) => { + let ret = pack.tokenizer[prop].apply(tokenizer, args) + if (ret === false) { + ret = prevTokenizer.apply(tokenizer, args) + } + return ret + } + } + opts.tokenizer = tokenizer + } + + // ==-- Parse WalkTokens extensions --== // + if (pack.walkTokens) { + const walkTokens = marked.defaults.walkTokens + opts.walkTokens = function (token) { + pack.walkTokens.call(this, token) + if (walkTokens) { + walkTokens.call(this, token) + } + } + } + + if (hasExtensions) { + opts.extensions = extensions + } + + marked.setOptions(opts) + }) +} + +/** + * Run callback for every token + */ + +marked.walkTokens = function (tokens, callback) { + for (const token of tokens) { + callback.call(marked, token) + switch (token.type) { + case "table": { + for (const cell of token.header) { + marked.walkTokens(cell.tokens, callback) + } + for (const row of token.rows) { + for (const cell of row) { + marked.walkTokens(cell.tokens, callback) + } + } + break + } + case "list": { + marked.walkTokens(token.items, callback) + break + } + default: { + if ( + marked.defaults.extensions && + marked.defaults.extensions.childTokens && + marked.defaults.extensions.childTokens[token.type] + ) { + // Walk any extensions + marked.defaults.extensions.childTokens[token.type].forEach(function ( + childTokens + ) { + marked.walkTokens(token[childTokens], callback) + }) + } else if (token.tokens) { + marked.walkTokens(token.tokens, callback) + } + } + } + } +} + +/** + * Parse Inline + * @param {string} src + */ +marked.parseInline = function (src, opt) { + // throw error in case of non string input + if (typeof src === "undefined" || src === null) { + throw new Error( + "marked.parseInline(): input parameter is undefined or null" + ) + } + if (typeof src !== "string") { + throw new Error( + "marked.parseInline(): input parameter is of type " + + Object.prototype.toString.call(src) + + ", string expected" + ) + } + + opt = merge({}, marked.defaults, opt || {}) + checkSanitizeDeprecation(opt) + + try { + const tokens = Lexer.lexInline(src, opt) + if (opt.walkTokens) { + marked.walkTokens(tokens, opt.walkTokens) + } + return Parser.parseInline(tokens, opt) + } catch (e) { + e.message += "\nPlease report this to https://github.com/markedjs/marked." + if (opt.silent) { + return ( + "

    An error occurred:

    " +
+        escape(e.message + "", true) +
+        "
    " + ) + } + throw e + } +} + +/** + * Expose + */ +marked.Parser = Parser +marked.parser = Parser.parse +marked.Renderer = Renderer +marked.TextRenderer = TextRenderer +marked.Lexer = Lexer +marked.lexer = Lexer.lex +marked.Tokenizer = Tokenizer +marked.Slugger = Slugger +marked.parse = marked + +const options = marked.options +const setOptions = marked.setOptions +const use = marked.use +const walkTokens = marked.walkTokens +const parseInline = marked.parseInline +const parse = marked +const parser = Parser.parse +const lexer = Lexer.lex + +const email = trigger.row +return marked(email.Message) From 335228e8782f9a9e7044c9869250c05d1c2817ba Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Mon, 19 Feb 2024 21:08:26 +0000 Subject: [PATCH 03/31] isolated VM test cases. --- .../src/jsRunner/tests/isolatedVM.spec.ts | 53 +++++++++++++++++++ .../src/jsRunner/tests/jsRunner.spec.ts | 21 ++++++++ 2 files changed, 74 insertions(+) diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts index 0bd400c8d7..fd26246de5 100644 --- a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -30,4 +30,57 @@ describe("Test isolated vm directly", () => { }) expect(result).toBe("

    dddd

    \n") }) + + it("handle a mapping case", async () => { + const context = { + data: { + data: { + searchProducts: { + results: [ + { imageLinks: ["_S/"] } + ] + } + } + } + } + const result = await compare(` + const dataUnnested = data.data.searchProducts.results + const emptyLink = "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRRC3hpq0MXqXssA28Lm5NrzcOYAyr--q3xyg&usqp=CAU" + let pImage = emptyLink + let sImage = emptyLink + let uImage = emptyLink + let lImage = emptyLink + let b1Image = emptyLink + let b2Image = emptyLink + + const dataTransformed = dataUnnested.map(x=> { + let imageLinks = x.imageLinks + for (let i = 0; i < imageLinks.length; i++){ + if(imageLinks[i].includes("_P/") || imageLinks[i].includes("_p/")){ + pImage = imageLinks[i] + } else if (imageLinks[i].includes("_S/") || imageLinks[i].includes("_s/")){ + sImage = imageLinks[i] + } else if (imageLinks[i].includes("_U/") || imageLinks[i].includes("_u/")){ + uImage = imageLinks[i] + } else if (imageLinks[i].includes("_L/") || imageLinks[i].includes("_l/")){ + lImage = imageLinks[i] + } else if (imageLinks[i].includes("_B/") || imageLinks[i].includes("_b/")){ + b1Image = imageLinks[i] + } else if (imageLinks[i].includes("_B2/") || imageLinks[i].includes("_b2/")){ + b2Image = imageLinks[i] + } + } + + const arrangedLinks = [pImage, sImage, uImage, lImage, b1Image, b2Image] + x.imageLinks = arrangedLinks + + return x + }) + + return dataTransformed + `, context) + expect(result).toBeDefined() + expect(result.length).toBe(1) + expect(result[0].imageLinks.length).toBe(6) + }) }) diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index 9bb9052eda..6dc6fd0887 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -144,5 +144,26 @@ describe("jsRunner (using isolated-vm)", () => { expect(result).toBeDefined() expect(result).toBe(3) }) + + it("should handle test case 4", async () => { + const context = { + "Time Sheets": ["a", "b"] + } + const result = await processJS(` + let hours = 0 + if (($("[Time Sheets]") != null) == true){ + for (i = 0; i < $("[Time Sheets]").length; i++){ + let hoursLogged = "Time Sheets." + i + ".Hours" + hours += $(hoursLogged) + } + return hours + } + if (($("[Time Sheets]") != null) == false){ + return hours + } + `, context) + expect(result).toBeDefined() + expect(result).toBe("0ab") + }) }) }) From fcea092667dd4a66de371ca4170a0405a64b7e5d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 20 Feb 2024 09:30:53 +0000 Subject: [PATCH 04/31] Remove link. --- packages/server/src/jsRunner/tests/isolatedVM.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts index fd26246de5..bd74940dd6 100644 --- a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -45,7 +45,7 @@ describe("Test isolated vm directly", () => { } const result = await compare(` const dataUnnested = data.data.searchProducts.results - const emptyLink = "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRRC3hpq0MXqXssA28Lm5NrzcOYAyr--q3xyg&usqp=CAU" + const emptyLink = "https://budibase.com" let pImage = emptyLink let sImage = emptyLink let uImage = emptyLink From 2e53be5bd841b12b46f79b429d642e600a6aa4b1 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 21 Feb 2024 16:13:43 +0000 Subject: [PATCH 05/31] add null check --- packages/builder/src/pages/builder/auth/_layout.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/auth/_layout.svelte b/packages/builder/src/pages/builder/auth/_layout.svelte index 8902e9ac61..1a40847aa0 100644 --- a/packages/builder/src/pages/builder/auth/_layout.svelte +++ b/packages/builder/src/pages/builder/auth/_layout.svelte @@ -10,7 +10,7 @@ $redirect("../") } - if ($admin.cloud && $admin?.checklist?.branding) { + if ($admin?.cloud && $admin?.checklist?.branding) { let url = new URL(window.location.href) let hostname = url.hostname let parts = hostname.split(".") From b6b39375a1a1fdb800aae8cc8e4ccb40ae59b512 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 21 Feb 2024 16:16:25 +0000 Subject: [PATCH 06/31] account portal ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 8c446c4ba3..92129b0d62 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 8c446c4ba385592127fa31755d3b64467b291882 +Subproject commit 92129b0d6251a7b4dabb367c15b09c079399b763 From fe28ae50e587302189998ed71f367e0c433ccca9 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 21 Feb 2024 16:18:12 +0000 Subject: [PATCH 07/31] ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 60e47a8249..336bf2184c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1 +Subproject commit 336bf2184cf632fdc2bffbad5628e8b15dd381bd From 25d59d0be0b1d9f74d353ae63077e41cd5b780f8 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 21 Feb 2024 16:31:32 +0000 Subject: [PATCH 08/31] Revert "ref" This reverts commit fe28ae50e587302189998ed71f367e0c433ccca9. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 336bf2184c..60e47a8249 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 336bf2184cf632fdc2bffbad5628e8b15dd381bd +Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1 From c94cf7c12c5273571a05f3d3a50c9c212c92bb63 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 21 Feb 2024 16:31:45 +0000 Subject: [PATCH 09/31] update --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 60e47a8249..336bf2184c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1 +Subproject commit 336bf2184cf632fdc2bffbad5628e8b15dd381bd From 1484170a221f62b35533f8e9ef1d47dfe5d4c78a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 21 Feb 2024 17:20:05 +0000 Subject: [PATCH 10/31] Fixing test case. --- .../src/jsRunner/tests/isolatedVM.spec.ts | 42 +++++++------------ 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts index bd74940dd6..7230f54714 100644 --- a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -1,31 +1,18 @@ import fs from "fs" import path from "path" -import { IsolatedVM, VM2 } from "../vm" +import { IsolatedVM } from "../vm" -function runJSWithIsolatedVM(script: string, context?: any) { +function runJSWithIsolatedVM(script: string, context: any) { const runner = new IsolatedVM() - if (context) { - runner.withContext(context) - } - return runner.execute(`(function(){\n${script}\n})();`) -} - -function runJSWithVM2(script: string, context?: any) { - const runner = new VM2(context) - return runner.execute(script) -} - -function compare(script: string, context?: any) { - const resultIsolated = runJSWithIsolatedVM(script, context) - const resultVM = runJSWithVM2(script, context) - expect(resultIsolated).toEqual(resultVM) - return resultIsolated + return runner.withContext(context, () => { + return runner.execute(`(function(){\n${script}\n})();`) + }) } describe("Test isolated vm directly", () => { it("should handle a very large file", () => { const marked = fs.readFileSync(path.join(__dirname, "marked.txt"), "utf-8") - const result = compare(marked, { + const result = runJSWithIsolatedVM(marked, { trigger: { row: { Message: "dddd" } }, }) expect(result).toBe("

    dddd

    \n") @@ -36,14 +23,13 @@ describe("Test isolated vm directly", () => { data: { data: { searchProducts: { - results: [ - { imageLinks: ["_S/"] } - ] - } - } - } + results: [{ imageLinks: ["_S/"] }], + }, + }, + }, } - const result = await compare(` + const result = await runJSWithIsolatedVM( + ` const dataUnnested = data.data.searchProducts.results const emptyLink = "https://budibase.com" let pImage = emptyLink @@ -78,7 +64,9 @@ describe("Test isolated vm directly", () => { }) return dataTransformed - `, context) + `, + context + ) expect(result).toBeDefined() expect(result.length).toBe(1) expect(result[0].imageLinks.length).toBe(6) From 6543207974409a0a644451c1a1af2163ecb9529b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 21 Feb 2024 18:36:48 +0000 Subject: [PATCH 11/31] Expanding test cases with a few more examples from queries, automations and formulas. --- .../src/jsRunner/tests/isolatedVM.spec.ts | 25 +++ .../src/jsRunner/tests/jsRunner.spec.ts | 194 +++++++++++++++++- 2 files changed, 212 insertions(+), 7 deletions(-) diff --git a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts index 7230f54714..a396fa895e 100644 --- a/packages/server/src/jsRunner/tests/isolatedVM.spec.ts +++ b/packages/server/src/jsRunner/tests/isolatedVM.spec.ts @@ -71,4 +71,29 @@ describe("Test isolated vm directly", () => { expect(result.length).toBe(1) expect(result[0].imageLinks.length).toBe(6) }) + + it("should handle automation script example", () => { + const context = { + steps: [{}, { response: "hello" }, { items: [{ rows: [{ a: 1 }] }] }], + } + const result = runJSWithIsolatedVM( + `const queryResults = steps[2].items; + + const intervals = steps[1].response; + const whereNoItemsReturned = []; + let index = 0; + + for (let queryResult of queryResults) { + if (queryResult.rows.length === 0) { + whereNoItemsReturned.push(intervals[index]); + } + index++; + } + + return whereNoItemsReturned; + `, + context + ) + expect(result).toEqual([]) + }) }) diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index 6dc6fd0887..ebe57b6ea3 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,7 +1,5 @@ import { validate as isValidUUID } from "uuid" import { processStringSync, encodeJSBinding } from "@budibase/string-templates" -import fs from "fs" -import path from "path" const { runJsHelpersTests } = require("@budibase/string-templates/test/utils") @@ -9,7 +7,9 @@ import tk from "timekeeper" import { init } from ".." import TestConfiguration from "../../tests/utilities/TestConfiguration" -tk.freeze("2021-01-21T12:00:00") +const DATE = "2021-01-21T12:00:00" + +tk.freeze(DATE) describe("jsRunner (using isolated-vm)", () => { const config = new TestConfiguration() @@ -92,7 +92,7 @@ describe("jsRunner (using isolated-vm)", () => { it("handle test case 2", async () => { const context = { - "Purchase Date": "2021-01-21T12:00:00", + "Purchase Date": DATE, } const result = await processJS( ` @@ -147,9 +147,10 @@ describe("jsRunner (using isolated-vm)", () => { it("should handle test case 4", async () => { const context = { - "Time Sheets": ["a", "b"] + "Time Sheets": ["a", "b"], } - const result = await processJS(` + const result = await processJS( + ` let hours = 0 if (($("[Time Sheets]") != null) == true){ for (i = 0; i < $("[Time Sheets]").length; i++){ @@ -161,9 +162,188 @@ describe("jsRunner (using isolated-vm)", () => { if (($("[Time Sheets]") != null) == false){ return hours } - `, context) + `, + context + ) expect(result).toBeDefined() expect(result).toBe("0ab") }) + + it("should handle test case 5", async () => { + const context = { + change: JSON.stringify({ a: 1, primaryDisplay: "a" }), + previous: JSON.stringify({ a: 2, primaryDisplay: "b" }), + } + const result = await processJS( + ` + let change = $("[change]") ? JSON.parse($("[change]")) : {} + let previous = $("[previous]") ? JSON.parse($("[previous]")) : {} + + function simplifyLink(originalKey, value, parent) { + if (Array.isArray(value)) { + if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) { + parent[originalKey] = value.map(link => link.primaryDisplay) + } + } + } + + for (let entry of Object.entries(change)) { + simplifyLink(entry[0], entry[1], change) + } + for (let entry of Object.entries(previous)) { + simplifyLink(entry[0], entry[1], previous) + } + + let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString())) + + delete diff.audit_change + delete diff.audit_previous + delete diff._id + delete diff._rev + delete diff.tableId + delete diff.audit + + for (let entry of Object.entries(diff)) { + simplifyLink(entry[0], entry[1], diff) + } + + return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}") + `, + context + ) + expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`) + }) + + it("should handle test case 6", async () => { + const context = { + "Join Date": DATE, + } + const result = await processJS( + ` + var rate = 5; + var today = new Date(); + + // comment + function monthDiff(dateFrom, dateTo) { + return dateTo.getMonth() - dateFrom.getMonth() + + (12 * (dateTo.getFullYear() - dateFrom.getFullYear())) + } + var serviceMonths = monthDiff( new Date($("[Join Date]")), today); + var serviceYears = serviceMonths / 12; + + if (serviceYears >= 1 && serviceYears < 5){ + rate = 10; + } + if (serviceYears >= 5 && serviceYears < 10){ + rate = 15; + } + if (serviceYears >= 10){ + rate = 15; + rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10); + } + return rate; + `, + context + ) + expect(result).toBe(10) + }) + + it("should handle test case 7", async () => { + const context = { + "P I": "Pass", + "PA I": "Pass", + "F I": "Fail", + "V I": "Pass", + } + const result = await processJS( + `if (($("[P I]") == "Pass") == true) + if (($("[ P I]") == "Pass") == true) + if (($("[F I]") == "Pass") == true) + if (($("[V I]") == "Pass") == true) + {return "Pass"} + + if (($("[PA I]") == "Fail") == true) + {return "Fail"} + if (($("[ P I]") == "Fail") == true) + {return "Fail"} + if (($("[F I]") == "Fail") == true) + {return "Fail"} + if (($("[V I]") == "Fail") == true) + {return "Fail"} + + else + {return ""}`, + context + ) + expect(result).toBe("Fail") + }) + + it("should handle test case 8", async () => { + const context = { + "T L": [{ Hours: 10 }], + "B H": 50, + } + const result = await processJS( + `var totalHours = 0; + if (($("[T L]") != null) == true){ + for (let i = 0; i < ($("[T L]").length); i++){ + var individualHours = "T L." + i + ".Hours"; + var hoursNum = Number($(individualHours)); + totalHours += hoursNum; + } + return totalHours.toFixed(2); + } + if (($("[T L]") != null) == false) { + return totalHours.toFixed(2); + } + `, + context + ) + expect(result).toBe("10.00") + }) + + it("should handle test case 9", async () => { + const context = { + "T L": [{ Hours: 10 }], + "B H": 50, + } + const result = await processJS( + `var totalHours = 0; + if (($("[T L]") != null) == true){ + for (let i = 0; i < ($("[T L]").length); i++){ + var individualHours = "T L." + i + ".Hours"; + var hoursNum = Number($(individualHours)); + totalHours += hoursNum; + } + return ($("[B H]") - totalHours).toFixed(2); + } + if (($("[T L]") != null) == false) { + return ($("[B H]") - totalHours).toFixed(2); + }`, + context + ) + expect(result).toBe("40.00") + }) + + it("should handle test case 10", async () => { + const context = { + "F F": [{ "F S": 10 }], + } + const result = await processJS( + `var rating = 0; + + if ($("[F F]") != null){ + for (i = 0; i < $("[F F]").length; i++){ + var individualRating = $("F F." + i + ".F S"); + rating += individualRating; + } + rating = (rating / $("[F F]").length); + } + return rating; + `, + context + ) + expect(result).toBe(10) + }) }) }) From 68719f1dda7126f391876b2a4f695e173310b565 Mon Sep 17 00:00:00 2001 From: Conor Webb Date: Thu, 22 Feb 2024 11:00:02 +0000 Subject: [PATCH 12/31] Remove Airtable from data source ui --- packages/server/src/api/controllers/integration.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/integration.ts b/packages/server/src/api/controllers/integration.ts index 23defac831..5ab29f0329 100644 --- a/packages/server/src/api/controllers/integration.ts +++ b/packages/server/src/api/controllers/integration.ts @@ -2,8 +2,16 @@ import { getDefinition, getDefinitions } from "../../integrations" import { BBContext } from "@budibase/types" export async function fetch(ctx: BBContext) { - ctx.status = 200 - ctx.body = await getDefinitions() + try { + const definitions = await getDefinitions() + delete definitions.AIRTABLE + ctx.status = 200 + ctx.body = definitions + } catch (error) { + console.error('Error fetching definitions:', error) + ctx.status = 500 + ctx.body = { error: 'Internal server error' } + } } export async function find(ctx: BBContext) { From b5d44dfcfd66f1a230b9b5473971fd33175cae82 Mon Sep 17 00:00:00 2001 From: Conor Webb Date: Thu, 22 Feb 2024 11:16:38 +0000 Subject: [PATCH 13/31] Fixed linting issue --- packages/server/src/api/controllers/integration.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/integration.ts b/packages/server/src/api/controllers/integration.ts index 5ab29f0329..935148ff2f 100644 --- a/packages/server/src/api/controllers/integration.ts +++ b/packages/server/src/api/controllers/integration.ts @@ -8,9 +8,9 @@ export async function fetch(ctx: BBContext) { ctx.status = 200 ctx.body = definitions } catch (error) { - console.error('Error fetching definitions:', error) + console.error("Error fetching definitions:", error) ctx.status = 500 - ctx.body = { error: 'Internal server error' } + ctx.body = { error: "Internal server error" } } } From fb1bfbdae8b9593bbb0db30bb01577e23aa4bd85 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 22 Feb 2024 13:55:26 +0100 Subject: [PATCH 14/31] Clean code --- packages/backend-core/src/middleware/joi-validator.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/middleware/joi-validator.ts b/packages/backend-core/src/middleware/joi-validator.ts index fcc8316886..ac8064a512 100644 --- a/packages/backend-core/src/middleware/joi-validator.ts +++ b/packages/backend-core/src/middleware/joi-validator.ts @@ -1,12 +1,12 @@ -import Joi, { ObjectSchema } from "joi" -import { BBContext } from "@budibase/types" +import Joi from "joi" +import { Ctx } from "@budibase/types" function validate( schema: Joi.ObjectSchema | Joi.ArraySchema, property: string ) { // Return a Koa middleware function - return (ctx: BBContext, next: any) => { + return (ctx: Ctx, next: any) => { if (!schema) { return next() } @@ -30,7 +30,6 @@ function validate( const { error } = schema.validate(params) if (error) { ctx.throw(400, `Invalid ${property} - ${error.message}`) - return } return next() } From 8e77365c1952a353328e87be13f6d63ac66f9c6c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 22 Feb 2024 13:55:38 +0100 Subject: [PATCH 15/31] Allow mocking multiple features --- packages/backend-core/tests/core/utilities/mocks/licenses.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 758fd6bf9a..1cbc282575 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -58,7 +58,7 @@ export const useCloudFree = () => { // FEATURES const useFeature = (feature: Feature) => { - const license = cloneDeep(UNLIMITED_LICENSE) + const license = cloneDeep(getCachedLicense() || UNLIMITED_LICENSE) const opts: UseLicenseOpts = { features: [feature], } From ea615f420ce5f2dc466b0a3a512eb25b4ea7071f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 22 Feb 2024 13:55:59 +0100 Subject: [PATCH 16/31] Add extra test --- .../src/api/routes/global/tests/scim.spec.ts | 20 +++++++++++++++++++ packages/worker/src/tests/api/groups.ts | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts index 56b7ca9f40..ae0abadc5d 100644 --- a/packages/worker/src/api/routes/global/tests/scim.spec.ts +++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts @@ -654,6 +654,26 @@ describe("scim", () => { totalResults: groupCount, }) }) + + it("can fetch groups even if internal groups exist", async () => { + mocks.licenses.useGroups() + await config.api.groups.saveGroup(structures.userGroups.userGroup()) + await config.api.groups.saveGroup(structures.userGroups.userGroup()) + + const response = await getScimGroups() + + expect(response).toEqual({ + Resources: expect.arrayContaining(groups), + itemsPerPage: 25, + schemas: ["urn:ietf:params:scim:api:messages:2.0:ListResponse"], + startIndex: 1, + totalResults: groupCount, + }) + + expect((await config.api.groups.fetch()).body.data).toHaveLength( + 25 + 2 // scim groups + internal groups + ) + }) }) }) diff --git a/packages/worker/src/tests/api/groups.ts b/packages/worker/src/tests/api/groups.ts index 91f7c92c7d..0b9081cc92 100644 --- a/packages/worker/src/tests/api/groups.ts +++ b/packages/worker/src/tests/api/groups.ts @@ -53,4 +53,12 @@ export class GroupsAPI extends TestAPI { .expect("Content-Type", /json/) .expect(200) } + + fetch = ({ expect } = { expect: 200 }) => { + return this.request + .get(`/api/global/groups`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(expect) + } } From 302c6d9e0b0438b610a13f54ad979c3f2571d3fe Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 22 Feb 2024 14:07:46 +0100 Subject: [PATCH 17/31] Update ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 60e47a8249..bd296586af 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1 +Subproject commit bd296586af0dc17081c613e73539bb34820ddf27 From de0414afbe6ff19dc27bc1bb0d5182ec51060d3e Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Thu, 22 Feb 2024 13:19:29 +0000 Subject: [PATCH 18/31] Update Svelte Major Version to 4 (#13036) * update svelte versions * temporarilly disable svelte warnings * change bbap version pointed to * remove sub module specifying svelte version * point at updated account-portal * add type declaration for svelte in the server * add svelte transformer for server jest tests * remove svelte from bbui package.json * get builder tests working * linting fixes * prettier fixes * fix server test * fix another server test * fix server serverlog test * account-portal-version * pin master version of pro * revert hbs doc file in server to use triple braces, ensures nothing gets escaped --- package.json | 2 +- packages/account-portal | 2 +- packages/bbui/package.json | 3 +- .../bbui/src/ActionButton/ActionButton.svelte | 1 + .../bbui/src/ActionMenu/ActionMenu.svelte | 2 + packages/bbui/src/Badge/Badge.svelte | 2 + .../bbui/src/ColorPicker/ColorPicker.svelte | 4 + .../src/DetailSummary/DetailSummary.svelte | 2 + packages/bbui/src/FancyForm/FancyField.svelte | 2 + packages/bbui/src/Form/Core/Combobox.svelte | 1 + packages/bbui/src/Form/Core/DatePicker.svelte | 3 + packages/bbui/src/Form/Core/Dropzone.svelte | 3 + .../bbui/src/Form/Core/EnvDropdown.svelte | 2 + packages/bbui/src/Form/Core/File.svelte | 2 + .../bbui/src/Form/Core/InputDropdown.svelte | 1 + packages/bbui/src/Form/Core/Picker.svelte | 1 + .../bbui/src/Form/Core/PickerDropdown.svelte | 1 + packages/bbui/src/Icon/Icon.svelte | 2 + .../bbui/src/IconPicker/IconPicker.svelte | 2 + .../src/IconSideNav/IconSideNavItem.svelte | 2 + packages/bbui/src/Input/CopyInput.svelte | 2 + packages/bbui/src/List/ListItem.svelte | 2 + packages/bbui/src/Menu/Item.svelte | 1 + packages/bbui/src/Modal/CustomContent.svelte | 2 + packages/bbui/src/Modal/Modal.svelte | 1 + .../OptionSelectDnD/OptionSelectDnD.svelte | 2 + .../bbui/src/Pagination/Pagination.svelte | 2 + packages/bbui/src/Popover/Popover.svelte | 1 + .../src/ProgressCircle/ProgressCircle.svelte | 2 + packages/bbui/src/SideNavigation/Item.svelte | 2 + .../bbui/src/StatusLight/StatusLight.svelte | 2 + .../bbui/src/Table/InternalRenderer.svelte | 2 + packages/bbui/src/Table/Table.svelte | 2 + packages/bbui/src/Tabs/Tab.svelte | 3 + packages/bbui/src/Tooltip/AbsTooltip.svelte | 1 + .../bbui/src/Tooltip/TooltipWrapper.svelte | 1 + packages/builder/package.json | 5 +- .../FlowChart/ActionModal.svelte | 2 + .../FlowChart/FlowChart.svelte | 2 + .../FlowChart/FlowItem.svelte | 2 + .../FlowChart/FlowItemHeader.svelte | 2 + .../CreateAutomationModal.svelte | 2 + .../automation/SetupPanel/SchemaSetup.svelte | 2 + .../DataTable/modals/CreateEditColumn.svelte | 2 + .../DataTable/modals/ExportModal.test.js | 1 + .../backend/TableNavigator/ListItem.svelte | 2 + .../commandPalette/CommandPalette.svelte | 2 + .../src/components/common/DashCard.svelte | 2 + .../common/Dropdowns/DropdownContainer.svelte | 2 + .../common/Dropdowns/DropdownItem.svelte | 1 + .../src/components/common/EditableIcon.svelte | 2 + .../src/components/common/Icons/Pencil.svelte | 2 + .../src/components/common/NavHeader.svelte | 1 + .../src/components/common/NavItem.svelte | 2 + .../common/bindings/BindingPicker.svelte | 3 + .../bindings/DrawerBindableCombobox.svelte | 2 + .../bindings/DrawerBindableInput.svelte | 2 + .../common/bindings/DrawerBindableSlot.svelte | 2 + .../common/bindings/ModalBindableInput.svelte | 2 + .../src/components/deploy/AppActions.svelte | 2 + .../src/components/design/Panel.svelte | 2 + .../ButtonActionDrawer.svelte | 3 + .../ButtonConfiguration.svelte | 2 + .../controls/ColumnEditor/ColumnDrawer.svelte | 1 + .../DataSourceCategory.svelte | 1 + .../DraggableList/DraggableList.svelte | 3 + .../FlatButtonGroup/FlatButton.svelte | 2 + .../controls/IconSelect/IconSelect.svelte | 3 + .../components/portal/page/SideNavItem.svelte | 2 + .../settings/UserGroupPicker.svelte | 2 + .../src/components/start/AppRow.svelte | 2 + .../components/start/ChooseIconModal.svelte | 2 + .../_components/BuilderSidePanel.svelte | 2 + .../_components/PreviewOverlay.svelte | 2 + .../data/_components/DatasourceOption.svelte | 2 + .../EditDatasourceConfigButton.svelte | 2 + .../data/datasource/bb_internal/index.svelte | 2 + .../index.svelte | 2 + .../Component/ConditionalUIDrawer.svelte | 1 + .../_components/Navigation/LinksDrawer.svelte | 1 + .../_components/Screen/AppThemeSelect.svelte | 2 + .../new/_components/NewComponentPanel.svelte | 2 + .../[screenId]/_components/AppPreview.svelte | 2 + .../ComponentScrollWrapper.svelte | 1 + .../ComponentList/ComponentTree.svelte | 2 + .../_components/ComponentList/index.svelte | 2 + .../ScreenList/RoleIndicator.svelte | 1 + .../NewScreen/DatasourceTemplateRow.svelte | 2 + .../design/_components/NewScreen/index.svelte | 2 + .../src/pages/builder/auth/forgot.svelte | 2 + .../builder/portal/_components/Logo.svelte | 2 + .../portal/_components/MobileMenu.svelte | 4 + .../portal/account/auditLogs/index.svelte | 2 + .../plugins/_components/PluginRow.svelte | 2 + .../portal/settings/auth/google.svelte | 2 + .../builder/portal/settings/auth/index.svelte | 2 + .../builder/portal/settings/auth/scim.svelte | 2 + .../portal/settings/diagnostics.svelte | 2 + .../email/_components/TemplateBindings.svelte | 2 + .../_components/OnboardingTypeModal.svelte | 2 + .../users/_components/PasswordModal.svelte | 2 + .../_components/RoleTableRenderer.svelte | 2 + .../builder/src/stores/builder/datasources.js | 3 +- .../builder/src/stores/builder/queries.js | 3 +- packages/builder/src/stores/builder/views.js | 2 +- .../builder/src/stores/builder/viewsV2.js | 2 +- packages/client/package.json | 25 +- .../src/components/app/Container.svelte | 2 + .../client/src/components/app/Icon.svelte | 2 + .../client/src/components/app/Layout.svelte | 2 + .../src/components/app/SpectrumCard.svelte | 4 + .../components/devtools/DevToolsStat.svelte | 2 + .../MissingRequiredAncestor.svelte | 2 + .../MissingRequiredSetting.svelte | 2 + .../components/preview/SettingsButton.svelte | 2 + packages/frontend-core/package.json | 3 +- .../grid/cells/AttachmentCell.svelte | 2 + .../src/components/grid/cells/GridCell.svelte | 2 + .../components/grid/cells/GutterCell.svelte | 2 + .../components/grid/cells/HeaderCell.svelte | 2 + .../components/grid/cells/LongFormCell.svelte | 2 + .../components/grid/cells/OptionsCell.svelte | 2 + .../grid/cells/RelationshipCell.svelte | 2 + .../grid/layout/ButtonColumn.svelte | 1 + .../src/components/grid/layout/Grid.svelte | 1 + .../components/grid/layout/GridBody.svelte | 2 + .../src/components/grid/layout/GridRow.svelte | 2 + .../grid/layout/GridScrollWrapper.svelte | 2 + .../grid/layout/NewColumnButton.svelte | 2 + .../src/components/grid/layout/NewRow.svelte | 2 + .../grid/layout/StickyColumn.svelte | 2 + .../grid/overlays/ResizeOverlay.svelte | 1 + .../grid/overlays/ScrollOverlay.svelte | 2 + packages/server/jest.config.ts | 14 + packages/server/package.json | 1 - packages/server/scripts/svelteTransformer.js | 11 + .../src/api/controllers/static/index.ts | 9 +- .../static/templates/BudibaseApp.d.svelte.ts | 11 + .../api/controllers/static/templates/app.hbs | 3 +- .../server/src/automations/tests/bash.spec.js | 18 +- .../automations/tests/executeQuery.spec.js | 28 +- .../src/automations/tests/serverLog.spec.js | 1 + packages/server/tsconfig.build.json | 3 +- scripts/build.js | 81 ++- yarn.lock | 506 ++++++++++++++---- 145 files changed, 777 insertions(+), 196 deletions(-) create mode 100644 packages/server/scripts/svelteTransformer.js create mode 100644 packages/server/src/api/controllers/static/templates/BudibaseApp.d.svelte.ts diff --git a/package.json b/package.json index 4407fd33f3..0a20f01d52 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", - "svelte": "3.49.0", + "svelte": "^4.2.10", "svelte-eslint-parser": "^0.33.1", "typescript": "5.2.2", "yargs": "^17.7.2" diff --git a/packages/account-portal b/packages/account-portal index 97329c0318..a851eeacab 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 97329c0318ef0f4bbbd2b9ce30d6976bc6505272 +Subproject commit a851eeacabfaad8bff6e781f5e5a62063cbc31f3 diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 78eed2b608..a1baa2a38b 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -24,8 +24,7 @@ "rollup": "^2.45.2", "rollup-plugin-postcss": "^4.0.0", "rollup-plugin-svelte": "^7.1.0", - "rollup-plugin-terser": "^7.0.2", - "svelte": "3.49.0" + "rollup-plugin-terser": "^7.0.2" }, "keywords": [ "svelte" diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 0e6ec3d155..c346e34d54 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -41,6 +41,7 @@ } + (showTooltip = true)} diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte index 08425e8f59..642ec4932a 100644 --- a/packages/bbui/src/ActionMenu/ActionMenu.svelte +++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte @@ -33,6 +33,8 @@ setContext("actionMenu", { show, hide }) + +
    diff --git a/packages/bbui/src/Badge/Badge.svelte b/packages/bbui/src/Badge/Badge.svelte index 8b54045297..e4ec7d4f33 100644 --- a/packages/bbui/src/Badge/Badge.svelte +++ b/packages/bbui/src/Badge/Badge.svelte @@ -13,6 +13,8 @@ export let hoverable = false + + + +
    + +
    diff --git a/packages/bbui/src/DetailSummary/DetailSummary.svelte b/packages/bbui/src/DetailSummary/DetailSummary.svelte index 2cbb6796f3..cbfdcbec9b 100644 --- a/packages/bbui/src/DetailSummary/DetailSummary.svelte +++ b/packages/bbui/src/DetailSummary/DetailSummary.svelte @@ -15,6 +15,8 @@ } + +
    {#if name}
    diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 455f4b38fb..798f486187 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -36,6 +36,8 @@ }) + +
    +
    + +
    {/key} {#if open} +
    {/if} diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index fa0be630ba..2bd95df516 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -137,6 +137,9 @@ } + + +
    {#if selectedImage} {#if gallery} diff --git a/packages/bbui/src/Form/Core/EnvDropdown.svelte b/packages/bbui/src/Form/Core/EnvDropdown.svelte index c690ffbc6b..ed5878d6b2 100644 --- a/packages/bbui/src/Form/Core/EnvDropdown.svelte +++ b/packages/bbui/src/Form/Core/EnvDropdown.svelte @@ -96,6 +96,8 @@ } + +
    + +
    {#if value}
    diff --git a/packages/bbui/src/Form/Core/InputDropdown.svelte b/packages/bbui/src/Form/Core/InputDropdown.svelte index 128353b7b9..c1bc2ac7e5 100644 --- a/packages/bbui/src/Form/Core/InputDropdown.svelte +++ b/packages/bbui/src/Form/Core/InputDropdown.svelte @@ -110,6 +110,7 @@ } +
    + +
    + +
    (showTooltip = true)} diff --git a/packages/bbui/src/IconPicker/IconPicker.svelte b/packages/bbui/src/IconPicker/IconPicker.svelte index b3cc72daa3..3cd7a16eb0 100644 --- a/packages/bbui/src/IconPicker/IconPicker.svelte +++ b/packages/bbui/src/IconPicker/IconPicker.svelte @@ -58,6 +58,8 @@ } + +
    (open = true)}>
    + +
    + +
    copyToClipboard(value)}> diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte index 28015c4c57..76b242cf9c 100644 --- a/packages/bbui/src/List/ListItem.svelte +++ b/packages/bbui/src/List/ListItem.svelte @@ -15,6 +15,8 @@ $: initials = avatar ? title?.[0] : null + +
    {#if icon} diff --git a/packages/bbui/src/Menu/Item.svelte b/packages/bbui/src/Menu/Item.svelte index ed759f5b10..05a33adda9 100644 --- a/packages/bbui/src/Menu/Item.svelte +++ b/packages/bbui/src/Menu/Item.svelte @@ -33,6 +33,7 @@ } +
  • + +
    Click me {remaining} diff --git a/packages/bbui/src/Modal/Modal.svelte b/packages/bbui/src/Modal/Modal.svelte index da97bf332e..f891d0584d 100644 --- a/packages/bbui/src/Modal/Modal.svelte +++ b/packages/bbui/src/Modal/Modal.svelte @@ -100,6 +100,7 @@ --> {#if visible} +
    + +
    + +