From c421499a4645166efd303da4d3bd0c7bbd7306e1 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 29 Nov 2021 17:21:30 +0000 Subject: [PATCH] cURL import working --- .../modals/ImportRestQueriesModal.svelte | 3 +- .../builder/src/stores/backend/queries.js | 5 + packages/server/package.json | 1 + .../src/api/controllers/query/import.ts | 71 +++++++++++- packages/server/yarn.lock | 103 +++++++++++++++++- 5 files changed, 174 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte index 8098e10fb4..4e8ba4fd77 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte @@ -56,8 +56,7 @@ datasourceId, } - const resp = await queries.import(body) - datasourceId = resp.datasourceId + await queries.import(body) // reload await datasources.fetch() diff --git a/packages/builder/src/stores/backend/queries.js b/packages/builder/src/stores/backend/queries.js index c03aea4052..6dfffc3d39 100644 --- a/packages/builder/src/stores/backend/queries.js +++ b/packages/builder/src/stores/backend/queries.js @@ -55,6 +55,11 @@ export function createQueriesStore() { }, import: async body => { const response = await api.post(`/api/queries/import`, body) + + if (response.status !== 200) { + throw new Error(response.message) + } + return response.json() }, select: query => { diff --git a/packages/server/package.json b/packages/server/package.json index 040febe02c..c73fe874ae 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -85,6 +85,7 @@ "bull": "^3.22.4", "chmodr": "1.2.0", "csvtojson": "2.0.10", + "curlconverter": "^3.21.0", "dotenv": "8.2.0", "download": "8.0.0", "fix-path": "3.0.0", diff --git a/packages/server/src/api/controllers/query/import.ts b/packages/server/src/api/controllers/query/import.ts index f860b28d8c..93f4b27ab1 100644 --- a/packages/server/src/api/controllers/query/import.ts +++ b/packages/server/src/api/controllers/query/import.ts @@ -2,6 +2,8 @@ import CouchDB from "../../../db" import { queryValidation } from "./validation" import { generateQueryID } from "../../../db/utils" import { Spec as Swagger2, Operation } from "swagger-schema-official" +const curlconverter = require("curlconverter") +import { URL } from "url" // { // "_id": "query_datasource_d62738f2d72a466997ffbf46f4952404_e7258ad382cd4c37961b81730633ff2d", @@ -90,15 +92,45 @@ interface DatasourceInfo { } const parseImportStrategy = (data: string): Strategy => { - return Strategy.SWAGGER2 + try { + const json = JSON.parse(data) + if (json.swagger === "2.0") { + return Strategy.CURL + } else if (json.openapi?.includes("3.0")) { + return Strategy.OPENAPI3 + } + } catch (jsonError) { + try { + parseCurl(data) + return Strategy.CURL + } catch (curlError) { + // do nothing + } + } + + throw new Error(`The import data could not be processed`) +} + +const processPath = (path: string): string => { + if (path?.startsWith("/")) { + return path.substring(1) + } + + return path } // SWAGGER const parseSwagger2Info = (swagger2: Swagger2): DatasourceInfo => { + const scheme = swagger2.schemes?.includes("https") ? "https" : "http" + const basePath = swagger2.basePath || "" + const host = swagger2.host || "" + const url = `${scheme}://${host}${basePath}` + const name = swagger2.info.title || "Swagger Import" + return { - url: "http://localhost:3000", - name: "swagger", + url: url, + name: name, defaultHeaders: [], } } @@ -150,16 +182,42 @@ const parseOpenAPI3Queries = (datasourceId: string, data: string): Query[] => { // CURL +const parseCurl = (data: string): any => { + const curlJson = curlconverter.toJsonString(data) + return JSON.parse(curlJson) +} + const parseCurlDatasourceInfo = (data: any): DatasourceInfo => { + const curl = parseCurl(data) + + const url = new URL(curl.url) + return { - url: "http://localhost:3000", - name: "swagger", + url: url.origin, + name: url.hostname, defaultHeaders: [], } } const parseCurlQueries = (datasourceId: string, data: string): Query[] => { - return [] + const curl = parseCurl(data) + + const url = new URL(curl.url) + const name = url.pathname + const path = url.pathname + const method = curl.method + const queryString = url.search + const headers = curl.headers + + const query = constructQuery( + datasourceId, + name, + method, + path, + queryString, + headers + ) + return [query] } const verbFromMethod = (method: string) => { @@ -183,6 +241,7 @@ const constructQuery = ( const queryVerb = verbFromMethod(method) const transformer = "return data" const schema = {} + path = processPath(path) const query: Query = { datasourceId, diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 3ef43963ef..997a856f6f 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1190,6 +1190,24 @@ dependencies: "@cspotcode/source-map-consumer" "0.8.0" +"@curlconverter/yargs-parser@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@curlconverter/yargs-parser/-/yargs-parser-0.0.1.tgz#62a360cea3d62b9b5805e61e8110cea98da8d140" + integrity sha512-DbEVRYqrorzwqc63MQ3RODflut1tNla8ZCKo1h83lF7+fbntgubZsDfRDBv5Lxj3vkKuvAolysNM2ekwJev8wA== + +"@curlconverter/yargs@^0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@curlconverter/yargs/-/yargs-0.0.2.tgz#a8ad4a2a2d0b0f897f8c117f664199165b59fd8a" + integrity sha512-Q1YEebpCY61kxme4wvU0/IN/uMBfG5pZOKCo9FU+w20ElPvN+eH2qEVbK1C12t3Tee3qeYLLEU6HkiUeO1gc4A== + dependencies: + "@curlconverter/yargs-parser" "^0.0.1" + cliui "^7.0.2" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.0" + y18n "^5.0.5" + "@elastic/elasticsearch@7.10.0": version "7.10.0" resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.10.0.tgz#da105a9c1f14146f9f2cab4e7026cb7949121b8d" @@ -2562,6 +2580,11 @@ resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.7.5.tgz#09fa51e356d07d0be200642b0e4f91d8e6dd408d" integrity sha512-V3BIhmY36fXZ1OtVcI9W+FxQqxVLsPKcNjWigIaa81dLC9IolJl5Mt4Cvhmr0flUnjSpTdrbMTSbXqYqV5dT6A== +a-sync-waterfall@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz#75b6b6aa72598b497a125e7a2770f14f4c8a1fa7" + integrity sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA== + abab@^2.0.0, abab@^2.0.3, abab@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" @@ -2896,6 +2919,11 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= +asap@^2.0.3: + version "2.0.6" + resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" + integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= + asn1@~0.2.3: version "0.2.6" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" @@ -3764,6 +3792,11 @@ commander@^2.19.0, commander@^2.5.0, commander@^2.8.1: resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== +commander@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" + integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== + commander@^7.1.0: version "7.2.0" resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" @@ -4016,6 +4049,19 @@ csvtojson@2.0.10: lodash "^4.17.3" strip-bom "^2.0.0" +curlconverter@^3.21.0: + version "3.21.0" + resolved "https://registry.yarnpkg.com/curlconverter/-/curlconverter-3.21.0.tgz#50b9568b8d24967ef55a0fcd872e7a18eea4cbc2" + integrity sha512-DXCnp1A/Xa69FujksUfdvWQFAnIn/C+4Wuv8t+UVdZkF/lY5bzj98GGKOGme7V/ckSHDLxE29Xp76sJ5Cpsp5A== + dependencies: + "@curlconverter/yargs" "^0.0.2" + cookie "^0.4.1" + jsesc "^3.0.2" + nunjucks "^3.2.3" + query-string "^7.0.1" + string.prototype.startswith "^1.0.0" + yamljs "^0.3.0" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -4537,7 +4583,7 @@ error-inject@^1.0.0: resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37" integrity sha1-4rPZG1Su1nLzCdlQ0VSFD6EdTzc= -es-abstract@^1.19.1: +es-abstract@^1.17.5, es-abstract@^1.19.1: version "1.19.1" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.19.1.tgz#d4885796876916959de78edaa0df456627115ec3" integrity sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w== @@ -5147,6 +5193,11 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +filter-obj@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/filter-obj/-/filter-obj-1.1.0.tgz#9b311112bc6c6127a16e016c6c5d7f19e0805c5b" + integrity sha1-mzERErxsYSehbgFsbF1/GeCAXFs= + find-up@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" @@ -7447,6 +7498,11 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +jsesc@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e" + integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g== + jsesc@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" @@ -8732,6 +8788,15 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +nunjucks@^3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/nunjucks/-/nunjucks-3.2.3.tgz#1b33615247290e94e28263b5d855ece765648a31" + integrity sha512-psb6xjLj47+fE76JdZwskvwG4MYsQKXUtMsPh6U0YMvmyjRtKRFcxnlXGWglNybtNTNVmGdp94K62/+NjF5FDQ== + dependencies: + a-sync-waterfall "^1.0.0" + asap "^2.0.3" + commander "^5.1.0" + nwsapi@^2.0.7, nwsapi@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7" @@ -9795,6 +9860,16 @@ query-string@^5.0.1: object-assign "^4.1.0" strict-uri-encode "^1.0.0" +query-string@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.0.1.tgz#45bd149cf586aaa582dffc7ec7a8ad97dd02f75d" + integrity sha512-uIw3iRvHnk9to1blJCG3BTc+Ro56CBowJXKmNNAm3RulvPBzWLRqKSiiDk+IplJhsydwtuNMHi8UGQFcCLVfkA== + dependencies: + decode-uri-component "^0.2.0" + filter-obj "^1.1.0" + split-on-first "^1.0.0" + strict-uri-encode "^2.0.0" + querystring@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" @@ -10677,6 +10752,11 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz#50c0d8c40a14ec1bf449bae69a0ea4685a9d9f95" integrity sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g== +split-on-first@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" + integrity sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw== + split-string@^3.0.1, split-string@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" @@ -10785,6 +10865,11 @@ strict-uri-encode@^1.0.0: resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713" integrity sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM= +strict-uri-encode@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" + integrity sha1-ucczDHBChi9rFC3CdLvMWGbONUY= + string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -10816,6 +10901,14 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2 is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" +string.prototype.startswith@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/string.prototype.startswith/-/string.prototype.startswith-1.0.0.tgz#92a361fb1ac172033d53eb1db3d659b0cfab6280" + integrity sha512-VHhsDkuf8gsw4JNRK9cIZjYe6r7PsVUutVohaBhqYAoPaRADoQH+mMgUg7Cs/TgQeDGEvI+PzPEMOdvdsCMvpg== + dependencies: + define-properties "^1.1.3" + es-abstract "^1.17.5" + string.prototype.trimend@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80" @@ -12137,6 +12230,14 @@ yaml@^1.10.2: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== +yamljs@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/yamljs/-/yamljs-0.3.0.tgz#dc060bf267447b39f7304e9b2bfbe8b5a7ddb03b" + integrity sha512-C/FsVVhht4iPQYXOInoxUM/1ELSf9EsgKH34FofQOp6hwCPrW4vG4w5++TED3xRUo8gD7l0P1J1dLlDYzODsTQ== + dependencies: + argparse "^1.0.7" + glob "^7.0.5" + yargs-parser@20.x, yargs-parser@^20.2.2: version "20.2.9" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"