From e59bdbbb8f1002121388e6f10a2315216b6e1c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 9 Jan 2024 14:44:58 +0000 Subject: [PATCH] Implement translation generator --- composer.json | 3 +- composer.lock | 97 +++++++++++++------ .../Tasks/DevGenerateTranslations.php | 83 +++++++++++++++- 3 files changed, 152 insertions(+), 31 deletions(-) diff --git a/composer.json b/composer.json index b92557d23..0d5e61063 100644 --- a/composer.json +++ b/composer.json @@ -90,7 +90,8 @@ "phpunit/phpunit": "9.5.20", "squizlabs/php_codesniffer": "^3.7", "swoole/ide-helper": "5.0.2", - "textalk/websocket": "1.5.7" + "textalk/websocket": "1.5.7", + "utopia-php/fetch": "0.1.*" }, "provide": { "ext-phpiredis": "*" diff --git a/composer.lock b/composer.lock index 5537294a7..f6cb205b0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "359b1e3bd27ac7362c6f8d145e64ae36", + "content-hash": "0922b311842c222911fce1ae3e3b352e", "packages": [ { "name": "adhocore/jwt", @@ -277,16 +277,16 @@ }, { "name": "chillerlan/php-settings-container", - "version": "2.1.4", + "version": "2.1.5", "source": { "type": "git", "url": "https://github.com/chillerlan/php-settings-container.git", - "reference": "1beb7df3c14346d4344b0b2e12f6f9a74feabd4a" + "reference": "f705310389264c3578fdd9ffb15aa2cd6d91772e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/1beb7df3c14346d4344b0b2e12f6f9a74feabd4a", - "reference": "1beb7df3c14346d4344b0b2e12f6f9a74feabd4a", + "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/f705310389264c3578fdd9ffb15aa2cd6d91772e", + "reference": "f705310389264c3578fdd9ffb15aa2cd6d91772e", "shasum": "" }, "require": { @@ -294,8 +294,10 @@ "php": "^7.4 || ^8.0" }, "require-dev": { - "phan/phan": "^5.3", - "phpunit/phpunit": "^9.5" + "phan/phan": "^5.4", + "phpcsstandards/php_codesniffer": "^3.8", + "phpmd/phpmd": "^2.13", + "phpunit/phpunit": "^9.6" }, "type": "library", "autoload": { @@ -337,7 +339,7 @@ "type": "ko_fi" } ], - "time": "2022-07-05T22:32:14+00:00" + "time": "2024-01-05T23:20:55+00:00" }, { "name": "dragonmantank/cron-expression", @@ -2072,12 +2074,12 @@ "version": "0.31.1", "source": { "type": "git", - "url": "https://github.com/utopia-php/framework.git", + "url": "https://github.com/utopia-php/http.git", "reference": "e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68", + "url": "https://api.github.com/repos/utopia-php/http/zipball/e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68", "reference": "e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68", "shasum": "" }, @@ -2107,8 +2109,8 @@ "upf" ], "support": { - "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.31.1" + "issues": "https://github.com/utopia-php/http/issues", + "source": "https://github.com/utopia-php/http/tree/0.31.1" }, "time": "2023-12-08T18:47:29+00:00" }, @@ -3489,25 +3491,27 @@ }, { "name": "nikic/php-parser", - "version": "v4.18.0", + "version": "v5.0.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999" + "reference": "4a21235f7e56e713259a6f76bf4b5ea08502b9dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/1bcbb2179f97633e98bbbc87044ee2611c7d7999", - "reference": "1bcbb2179f97633e98bbbc87044ee2611c7d7999", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/4a21235f7e56e713259a6f76bf4b5ea08502b9dc", + "reference": "4a21235f7e56e713259a6f76bf4b5ea08502b9dc", "shasum": "" }, "require": { + "ext-ctype": "*", + "ext-json": "*", "ext-tokenizer": "*", - "php": ">=7.0" + "php": ">=7.4" }, "require-dev": { "ircmaxell/php-yacc": "^0.0.7", - "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0" + "phpunit/phpunit": "^7.0 || ^8.0 || ^9.0" }, "bin": [ "bin/php-parse" @@ -3515,7 +3519,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "4.9-dev" + "dev-master": "5.0-dev" } }, "autoload": { @@ -3539,9 +3543,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.18.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.0" }, - "time": "2023-12-10T21:03:43+00:00" + "time": "2024-01-07T17:17:35+00:00" }, { "name": "phar-io/manifest", @@ -3893,16 +3897,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.24.5", + "version": "1.25.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "fedf211ff14ec8381c9bf5714e33a7a552dd1acc" + "reference": "bd84b629c8de41aa2ae82c067c955e06f1b00240" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/fedf211ff14ec8381c9bf5714e33a7a552dd1acc", - "reference": "fedf211ff14ec8381c9bf5714e33a7a552dd1acc", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/bd84b629c8de41aa2ae82c067c955e06f1b00240", + "reference": "bd84b629c8de41aa2ae82c067c955e06f1b00240", "shasum": "" }, "require": { @@ -3934,9 +3938,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.5" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.25.0" }, - "time": "2023-12-16T09:33:33+00:00" + "time": "2024-01-04T17:06:16+00:00" }, { "name": "phpunit/php-code-coverage", @@ -5821,6 +5825,45 @@ } ], "time": "2023-11-21T18:54:41+00:00" + }, + { + "name": "utopia-php/fetch", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, + "time": "2023-10-10T11:58:32+00:00" } ], "aliases": [], diff --git a/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php b/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php index 4db6ec11b..fca430d8d 100644 --- a/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php +++ b/src/Appwrite/Platform/Tasks/DevGenerateTranslations.php @@ -2,11 +2,17 @@ namespace Appwrite\Platform\Tasks; +use Exception; use Utopia\CLI\Console; +use Utopia\Fetch\Client; use Utopia\Platform\Action; +use Utopia\Validator\Boolean; +use Utopia\Validator\Text; class DevGenerateTranslations extends Action { + private string $apiKey = ''; + public static function getName(): string { return 'dev-generate-translations'; @@ -16,11 +22,82 @@ class DevGenerateTranslations extends Action { $this ->desc('Generate translations in all languages') - ->callback(fn () => $this->action()); + ->param('dry-run', true, new Boolean(true), 'If action should do a dry run. Dry run does not write into files', true) + ->param('api-key', '', new Text(256), 'Open AI API key. Only used during non-dry runs to generate translations.', true) + ->callback(fn ($dryRun, $apiKey) => $this->action($dryRun, $apiKey)); } - public function action(): void + public function action(bool|string $dryRun, string $apiKey): void { - Console::info("Empty command"); + $dryRun = \strval($dryRun) === 'true'; + + Console::info("Started"); + + if(!$dryRun && empty($apiKey)) { + Console::error("Please specify --api-key='OPEN_AI_API_KEY' or run with --dry-run"); + return; + } + + $this->apiKey = $apiKey; + + $dir = __DIR__ . '/../../../../app/config/locale/translations'; + $mainFile = 'en.json'; + + $mainJson = \json_decode(\file_get_contents($dir . '/' . $mainFile), true); + $mainKeys = \array_keys($mainJson); + + $files = array_diff(scandir($dir), array('.', '..', $mainFile)); + + foreach ($files as $file) { + $fileJson = \json_decode(\file_get_contents($dir . '/' . $file), true); + $fileKeys = \array_keys($fileJson); + + foreach ($mainKeys as $key) { + if(!(\in_array($key, $fileKeys))) { + if($dryRun) { + Console::warning("{$file} missing translation for {$key}"); + } else { + $language = \explode('.', $file)[0]; + $translation = $this->generateTranslation($language, $mainJson[$key]); + + $json = \json_decode(\file_get_contents($dir . '/' . $file), true); + $json[$key] = $translation; + \file_put_contents($dir . '/' . $file, \json_encode($json, JSON_PRETTY_PRINT|JSON_UNESCAPED_UNICODE|JSON_UNESCAPED_SLASHES|0)); + + Console::success("Generated {$key} for {$language}"); + } + } + } + } + + Console::info("Done"); + } + + private function generateTranslation(string $targetLanguage, string $enTranslation): string + { + $response = Client::fetch('https://api.openai.com/v1/chat/completions', [ + 'content-type' => Client::CONTENT_TYPE_APPLICATION_JSON, + 'Authorization' => 'Bearer ' . $this->apiKey + ], Client::METHOD_POST, [ + 'model' => 'gpt-4-1106-preview', // https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo + 'messages' => [ + [ + 'role' => 'system', + 'content' => "Please translate the message user provides from English language to target language. Target language is language of country with country code {$targetLanguage}. Do not translate text inside {{ and }} placeholders. Provide only translated text." + ], + [ + 'role' => 'user', + 'content' => $enTranslation + ] + ] + ]); + + $body = \json_decode($response->getBody(), true); + + if($response->getStatusCode() >= 400) { + throw new Exception($response->getBody()); + } + + return $body['choices'][0]['message']['content']; } }