From a6e9b78b09ed159497795ddf328759f4f6e51739 Mon Sep 17 00:00:00 2001 From: Christopher Schnick Date: Wed, 4 Jan 2023 05:23:57 +0100 Subject: [PATCH] Refactor shells --- .../impl/LocalProcessControlProvider.java | 1 - .../io/xpipe/core/impl/OutputStreamStore.java | 5 ++ .../core/process/CommandProcessControl.java | 6 +- .../io/xpipe/core/process/ProcessControl.java | 2 + .../java/io/xpipe/core/process/ShellType.java | 39 ++++----- .../io/xpipe/core/store/MachineStore.java | 8 +- .../java/io/xpipe/core/util/Deobfuscator.java | 2 +- .../java/io/xpipe/core/util/XPipeSession.java | 53 ++++++++++++ .../xpipe/core/util/XPipeTempDirectory.java | 6 +- .../extension/util/ApplicationHelper.java | 2 +- .../extension/util/DaemonExtensionTest.java | 4 + .../extension/util/ExecScriptHelper.java | 52 ------------ .../extension/util/LocalExtensionTest.java | 4 + .../io/xpipe/extension/util/ScriptHelper.java | 80 +++++++++++++++++++ .../io/xpipe/extension/util/Validators.java | 17 ++-- .../resources/lang/translations_en.properties | 2 +- 16 files changed, 186 insertions(+), 97 deletions(-) create mode 100644 core/src/main/java/io/xpipe/core/util/XPipeSession.java delete mode 100644 extension/src/main/java/io/xpipe/extension/util/ExecScriptHelper.java create mode 100644 extension/src/main/java/io/xpipe/extension/util/ScriptHelper.java diff --git a/core/src/main/java/io/xpipe/core/impl/LocalProcessControlProvider.java b/core/src/main/java/io/xpipe/core/impl/LocalProcessControlProvider.java index 4692a058..fca96a9f 100644 --- a/core/src/main/java/io/xpipe/core/impl/LocalProcessControlProvider.java +++ b/core/src/main/java/io/xpipe/core/impl/LocalProcessControlProvider.java @@ -1,7 +1,6 @@ package io.xpipe.core.impl; import io.xpipe.core.process.ShellProcessControl; -import io.xpipe.core.process.ShellTypes; import java.util.ServiceLoader; diff --git a/core/src/main/java/io/xpipe/core/impl/OutputStreamStore.java b/core/src/main/java/io/xpipe/core/impl/OutputStreamStore.java index 1c93ecfe..6353c518 100644 --- a/core/src/main/java/io/xpipe/core/impl/OutputStreamStore.java +++ b/core/src/main/java/io/xpipe/core/impl/OutputStreamStore.java @@ -14,6 +14,11 @@ public class OutputStreamStore implements StreamDataStore { this.out = out; } + @Override + public boolean isContentExclusivelyAccessible() { + return true; + } + @Override public DataFlow getFlow() { return DataFlow.OUTPUT; diff --git a/core/src/main/java/io/xpipe/core/process/CommandProcessControl.java b/core/src/main/java/io/xpipe/core/process/CommandProcessControl.java index 83df9e65..68b92e18 100644 --- a/core/src/main/java/io/xpipe/core/process/CommandProcessControl.java +++ b/core/src/main/java/io/xpipe/core/process/CommandProcessControl.java @@ -7,6 +7,8 @@ import java.util.function.Consumer; public interface CommandProcessControl extends ProcessControl { + @Override + public CommandProcessControl sensitive(); CommandProcessControl complex(); default InputStream startExternalStdout() throws Exception { @@ -64,9 +66,7 @@ public interface CommandProcessControl extends ProcessControl { String readOnlyStdout() throws Exception; - public default void discardOrThrow() throws Exception { - readOrThrow(); - } + public void discardOrThrow() throws Exception; void accumulateStdout(Consumer con); diff --git a/core/src/main/java/io/xpipe/core/process/ProcessControl.java b/core/src/main/java/io/xpipe/core/process/ProcessControl.java index 29092ea2..76d616af 100644 --- a/core/src/main/java/io/xpipe/core/process/ProcessControl.java +++ b/core/src/main/java/io/xpipe/core/process/ProcessControl.java @@ -8,6 +8,8 @@ import java.nio.charset.Charset; public interface ProcessControl extends Closeable, AutoCloseable { + ProcessControl sensitive(); + String prepareTerminalOpen() throws Exception; void closeStdin() throws IOException; diff --git a/core/src/main/java/io/xpipe/core/process/ShellType.java b/core/src/main/java/io/xpipe/core/process/ShellType.java index fb0e9fb9..f7f8abe4 100644 --- a/core/src/main/java/io/xpipe/core/process/ShellType.java +++ b/core/src/main/java/io/xpipe/core/process/ShellType.java @@ -5,6 +5,7 @@ import io.xpipe.core.charsetter.NewLine; import java.nio.charset.Charset; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @JsonTypeInfo( @@ -15,15 +16,11 @@ public interface ShellType { String getScriptFileEnding(); - default String commandWithVariable(String key, String value, String command) { - return joinCommands(getSetVariableCommand(key, value), command); - } + String addInlineVariablesToCommand(Map variables, String command); String getPauseCommand(); - String createInitFileContent(String command); - - String getTerminalFileOpenCommand(String file); + String prepareScriptContent(String content); default String flatten(List command) { return command.stream() @@ -35,12 +32,6 @@ public interface ShellType { .collect(Collectors.joining(" ")); } - default String joinCommands(String... s) { - return String.join(getConcatenationOperator(), s); - } - - void elevate(ShellProcessControl control, String command, String displayCommand) throws Exception; - default String getExitCommand() { return "exit"; } @@ -61,11 +52,11 @@ public interface ShellType { return getEchoCommand(s, false); } + String getSetVariableCommand(String variable, String value); + + String getEchoCommand(String s, boolean toErrorStream); - String queryShellProcessId(ShellProcessControl control) throws Exception; - - String getSetVariableCommand(String variableName, String value); default String getPrintVariableCommand(String name) { return getPrintVariableCommand("", name); @@ -79,19 +70,21 @@ public interface ShellType { List executeCommandListWithShell(String cmd); - List createMkdirsCommand(String dirs); + List getMkdirsCommand(String dirs); - String createFileReadCommand(String file); + String getFileReadCommand(String file); - String createFileWriteCommand(String file); + String getStreamFileWriteCommand(String file); - String createFileDeleteCommand(String file); + String getSimpleFileWriteCommand(String content, String file); - String createFileExistsCommand(String file); + String getFileDeleteCommand(String file); - String createFileTouchCommand(String file); + String getFileExistsCommand(String file); - String createWhichCommand(String executable); + String getFileTouchCommand(String file); + + String getWhichCommand(String executable); Charset determineCharset(ShellProcessControl control) throws Exception; @@ -103,5 +96,5 @@ public interface ShellType { String getExecutable(); - boolean echoesInput(); + boolean doesRepeatInput(); } diff --git a/core/src/main/java/io/xpipe/core/store/MachineStore.java b/core/src/main/java/io/xpipe/core/store/MachineStore.java index 0b2c726f..b59de4d0 100644 --- a/core/src/main/java/io/xpipe/core/store/MachineStore.java +++ b/core/src/main/java/io/xpipe/core/store/MachineStore.java @@ -12,21 +12,21 @@ public interface MachineStore extends FileSystemStore, ShellStore { @Override public default InputStream openInput(String file) throws Exception { return create().command(proc -> proc.getShellType() - .createFileReadCommand(proc.getOsType().normalizeFileName(file))) + .getFileReadCommand(proc.getOsType().normalizeFileName(file))) .startExternalStdout(); } @Override public default OutputStream openOutput(String file) throws Exception { return create().command(proc -> proc.getShellType() - .createFileWriteCommand(proc.getOsType().normalizeFileName(file))) + .getStreamFileWriteCommand(proc.getOsType().normalizeFileName(file))) .startExternalStdin(); } @Override public default boolean exists(String file) throws Exception { try (var pc = create().command(proc -> proc.getShellType() - .createFileExistsCommand(proc.getOsType().normalizeFileName(file))) + .getFileExistsCommand(proc.getOsType().normalizeFileName(file))) .start()) { return pc.discardAndCheckExit(); } @@ -36,7 +36,7 @@ public interface MachineStore extends FileSystemStore, ShellStore { public default boolean mkdirs(String file) throws Exception { try (var pc = create().command(proc -> proc.getShellType() .flatten(proc.getShellType() - .createMkdirsCommand(proc.getOsType().normalizeFileName(file)))) + .getMkdirsCommand(proc.getOsType().normalizeFileName(file)))) .start()) { return pc.discardAndCheckExit(); } diff --git a/core/src/main/java/io/xpipe/core/util/Deobfuscator.java b/core/src/main/java/io/xpipe/core/util/Deobfuscator.java index e766f836..d2d058c2 100644 --- a/core/src/main/java/io/xpipe/core/util/Deobfuscator.java +++ b/core/src/main/java/io/xpipe/core/util/Deobfuscator.java @@ -105,6 +105,6 @@ public class Deobfuscator { } var t = ShellTypes.getPlatformDefault(); - return LocalProcess.executeSimpleBooleanCommand(t.createWhichCommand("retrace." + t.getScriptFileEnding())); + return LocalProcess.executeSimpleBooleanCommand(t.getWhichCommand("retrace." + t.getScriptFileEnding())); } } diff --git a/core/src/main/java/io/xpipe/core/util/XPipeSession.java b/core/src/main/java/io/xpipe/core/util/XPipeSession.java new file mode 100644 index 00000000..08006984 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/util/XPipeSession.java @@ -0,0 +1,53 @@ +package io.xpipe.core.util; + +import io.xpipe.core.process.OsType; +import lombok.Value; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.UUID; + +@Value +public class XPipeSession { + + boolean isNewSystemSession; + + /** + * Unique identifier that resets on every X-Pipe restart. + */ + UUID sessionId; + + /** + * Unique identifier that resets on every X-Pipe update. + */ + UUID buildSessionId; + + /** + * Unique identifier that resets on system restarts. + */ + UUID systemSessionId; + + private static XPipeSession INSTANCE; + + public static void init(UUID buildSessionId) throws Exception { + var sessionFile = XPipeTempDirectory.getLocal().resolve("xpipe_session"); + var isNew = !Files.exists(sessionFile); + var systemSessionId = isNew ? UUID.randomUUID() : UUID.fromString(Files.readString(sessionFile)); + + if (OsType.getLocal().equals(OsType.WINDOWS)) { + var pf = Path.of("C:\\pagefile.sys"); + BasicFileAttributes attr = Files.readAttributes(pf, BasicFileAttributes.class); + var timeUuid = UUID.nameUUIDFromBytes(attr.creationTime().toInstant().toString().getBytes()); + isNew = isNew && timeUuid.equals(systemSessionId); + systemSessionId = timeUuid; + } + + Files.writeString(sessionFile, systemSessionId.toString()); + INSTANCE = new XPipeSession(isNew, UUID.randomUUID(), buildSessionId, systemSessionId); + } + + public static XPipeSession get() { + return INSTANCE; + } +} diff --git a/core/src/main/java/io/xpipe/core/util/XPipeTempDirectory.java b/core/src/main/java/io/xpipe/core/util/XPipeTempDirectory.java index a9a46f4a..5f8a4c1f 100644 --- a/core/src/main/java/io/xpipe/core/util/XPipeTempDirectory.java +++ b/core/src/main/java/io/xpipe/core/util/XPipeTempDirectory.java @@ -20,8 +20,8 @@ public class XPipeTempDirectory { var base = proc.getOsType().getTempDirectory(proc); var dir = FileNames.join(base, "xpipe"); - if (!proc.executeBooleanSimpleCommand(proc.getShellType().createFileExistsCommand(dir))) { - proc.executeSimpleCommand(proc.getShellType().flatten(proc.getShellType().createMkdirsCommand(dir)), "Unable to access or create temporary directory " + dir); + if (!proc.executeBooleanSimpleCommand(proc.getShellType().getFileExistsCommand(dir))) { + proc.executeSimpleCommand(proc.getShellType().flatten(proc.getShellType().getMkdirsCommand(dir)), "Unable to access or create temporary directory " + dir); if (proc.getOsType().equals(OsType.LINUX) || proc.getOsType().equals(OsType.MAC)) { proc.executeSimpleCommand("(chmod -f 777 \"" + dir + "\""); @@ -33,7 +33,7 @@ public class XPipeTempDirectory { public static void clear(ShellProcessControl proc) throws Exception { var dir = get(proc); - if (!proc.executeBooleanSimpleCommand(proc.getShellType().createFileDeleteCommand(dir))) { + if (!proc.executeBooleanSimpleCommand(proc.getShellType().getFileDeleteCommand(dir))) { throw new IOException("Unable to delete temporary directory " + dir); } } diff --git a/extension/src/main/java/io/xpipe/extension/util/ApplicationHelper.java b/extension/src/main/java/io/xpipe/extension/util/ApplicationHelper.java index f15d89f0..a9a2d944 100644 --- a/extension/src/main/java/io/xpipe/extension/util/ApplicationHelper.java +++ b/extension/src/main/java/io/xpipe/extension/util/ApplicationHelper.java @@ -7,7 +7,7 @@ import java.io.IOException; public class ApplicationHelper { public static boolean isInPath(ShellProcessControl processControl, String executable) throws Exception { - return processControl.executeBooleanSimpleCommand(processControl.getShellType().createWhichCommand(executable)); + return processControl.executeBooleanSimpleCommand(processControl.getShellType().getWhichCommand(executable)); } public static void checkSupport(ShellProcessControl processControl, String executable, String displayName) throws Exception { diff --git a/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java index 58c70149..01d2b07a 100644 --- a/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java @@ -4,10 +4,13 @@ import io.xpipe.api.DataSource; import io.xpipe.beacon.BeaconDaemonController; import io.xpipe.core.store.DataStore; import io.xpipe.core.util.JacksonMapper; +import io.xpipe.core.util.XPipeSession; import io.xpipe.extension.XPipeServiceProviders; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import java.util.UUID; + public class DaemonExtensionTest extends ExtensionTest { public static DataSource getSource(String type, DataStore store) { @@ -26,6 +29,7 @@ public class DaemonExtensionTest extends ExtensionTest { public static void setup() throws Exception { JacksonMapper.initModularized(ModuleLayer.boot()); XPipeServiceProviders.load(ModuleLayer.boot()); + XPipeSession.init(UUID.randomUUID()); BeaconDaemonController.start(); } diff --git a/extension/src/main/java/io/xpipe/extension/util/ExecScriptHelper.java b/extension/src/main/java/io/xpipe/extension/util/ExecScriptHelper.java deleted file mode 100644 index 2cdd6e1d..00000000 --- a/extension/src/main/java/io/xpipe/extension/util/ExecScriptHelper.java +++ /dev/null @@ -1,52 +0,0 @@ -package io.xpipe.extension.util; - -import io.xpipe.core.impl.FileNames; -import io.xpipe.core.process.ShellProcessControl; -import io.xpipe.core.store.ShellStore; -import io.xpipe.core.util.XPipeTempDirectory; -import lombok.SneakyThrows; - -import java.util.Objects; - -public class ExecScriptHelper { - - public static int getConnectionHash(String command) { - return Math.abs(Objects.hash(command)); - } - - @SneakyThrows - public static String createLocalExecScript(String content) { - try (var l = ShellStore.local().create().start()) { - return createExecScript(l, content); - } - } - - @SneakyThrows - public static String createExecScript(ShellProcessControl processControl, String content) { - var fileName = "exec-" + getConnectionHash(content); - content = processControl.getShellType().createInitFileContent(content); - var temp = XPipeTempDirectory.get(processControl); - var file = FileNames.join( - temp, fileName + "." + processControl.getShellType().getScriptFileEnding()); - - if (processControl.executeBooleanSimpleCommand(processControl.getShellType().createFileExistsCommand(file))) { - return file; - } - - try (var c = processControl.command(processControl.getShellType() - .joinCommands( - processControl.getShellType().createFileWriteCommand(file), - processControl.getShellType().getMakeExecutableCommand(file) - )) - .start()) { - c.discardOut(); - c.discardErr(); - c.getStdin().write(content.getBytes(processControl.getCharset())); - c.closeStdin(); - } - - processControl.restart(); - - return file; - } -} diff --git a/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java index 0a1b43ab..0291dd9b 100644 --- a/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java @@ -1,14 +1,18 @@ package io.xpipe.extension.util; import io.xpipe.core.util.JacksonMapper; +import io.xpipe.core.util.XPipeSession; import io.xpipe.extension.XPipeServiceProviders; import org.junit.jupiter.api.BeforeAll; +import java.util.UUID; + public class LocalExtensionTest extends ExtensionTest { @BeforeAll public static void setup() throws Exception { JacksonMapper.initModularized(ModuleLayer.boot()); XPipeServiceProviders.load(ModuleLayer.boot()); + XPipeSession.init(UUID.randomUUID()); } } diff --git a/extension/src/main/java/io/xpipe/extension/util/ScriptHelper.java b/extension/src/main/java/io/xpipe/extension/util/ScriptHelper.java new file mode 100644 index 00000000..07f6e11e --- /dev/null +++ b/extension/src/main/java/io/xpipe/extension/util/ScriptHelper.java @@ -0,0 +1,80 @@ +package io.xpipe.extension.util; + +import io.xpipe.core.impl.FileNames; +import io.xpipe.core.process.ShellProcessControl; +import io.xpipe.core.process.ShellType; +import io.xpipe.core.store.ShellStore; +import io.xpipe.core.util.SecretValue; +import io.xpipe.core.util.XPipeSession; +import io.xpipe.core.util.XPipeTempDirectory; +import io.xpipe.extension.event.TrackEvent; +import lombok.SneakyThrows; + +import java.util.Objects; + +public class ScriptHelper { + + public static int getConnectionHash(String command) { + return Math.abs(Objects.hash(command, XPipeSession.get().getSystemSessionId())); + } + + @SneakyThrows + public static String createLocalExecScript(String content) { + try (var l = ShellStore.local().create().start()) { + return createExecScript(l, content, false); + } + } + + @SneakyThrows + public static String createExecScript(ShellProcessControl processControl, String content, boolean restart) { + var fileName = "exec-" + getConnectionHash(content); + ShellType type = processControl.getShellType(); + var temp = XPipeTempDirectory.get(processControl); + var file = FileNames.join(temp, fileName + "." + type.getScriptFileEnding()); + return createExecScript(processControl, file, content, restart); + } + + @SneakyThrows + private static String createExecScript(ShellProcessControl processControl, String file, String content, boolean restart) { + ShellType type = processControl.getShellType(); + content = type.prepareScriptContent(content); + + if (processControl.executeBooleanSimpleCommand(type.getFileExistsCommand(file))) { + return file; + } + + TrackEvent.withTrace("proc", "Writing exec script") + .tag("file", file) + .tag("content", content) + .handle(); + + processControl.executeSimpleCommand(type.getFileTouchCommand(file), "Failed to create script " + file); + processControl.executeSimpleCommand(type.getMakeExecutableCommand(file), "Failed to make script " + file + " executable"); + + if (!content.contains("\n")) { + processControl.executeSimpleCommand(type.getSimpleFileWriteCommand(content, file)); + return file; + } + + try (var c = processControl.command(type.getStreamFileWriteCommand(file)).start()) { + c.discardOut(); + c.discardErr(); + c.getStdin().write(content.getBytes(processControl.getCharset())); + c.closeStdin(); + } + + if (restart) { + processControl.restart(); + } + + return file; + } + + @SneakyThrows + public static String createAskPassScript(SecretValue pass, ShellProcessControl parent, ShellType type, boolean restart) { + var content = type.getScriptEchoCommand(pass.getSecretValue()); + var temp = XPipeTempDirectory.get(parent); + var file = FileNames.join(temp, "askpass-" + getConnectionHash(content) + "." + type.getScriptFileEnding()); + return createExecScript(parent,file, content, restart); + } +} diff --git a/extension/src/main/java/io/xpipe/extension/util/Validators.java b/extension/src/main/java/io/xpipe/extension/util/Validators.java index 4d46deb8..85c5d49d 100644 --- a/extension/src/main/java/io/xpipe/extension/util/Validators.java +++ b/extension/src/main/java/io/xpipe/extension/util/Validators.java @@ -3,33 +3,34 @@ package io.xpipe.extension.util; import io.xpipe.core.impl.LocalStore; import io.xpipe.core.store.DataStore; import io.xpipe.core.store.ShellStore; +import io.xpipe.core.util.ValidationException; import io.xpipe.extension.I18n; import java.util.function.Predicate; public class Validators { - public static void nonNull(Object object, String name) { + public static void nonNull(Object object, String name) throws ValidationException { if (object == null) { - throw new IllegalArgumentException(I18n.get("extension.null", name)); + throw new ValidationException(I18n.get("extension.mustNotBeEmpty", name)); } } - public static void notEmpty(String string, String name) { + public static void notEmpty(String string, String name) throws ValidationException { if (string.trim().length() == 0) { - throw new IllegalArgumentException(I18n.get("extension.empty", name)); + throw new ValidationException(I18n.get("extension.mustNotBeEmpty", name)); } } - public static void namedStoreExists(DataStore store, String name) { + public static void namedStoreExists(DataStore store, String name) throws ValidationException { if (!XPipeDaemon.getInstance().getNamedStores().contains(store) && !(store instanceof LocalStore)) { - throw new IllegalArgumentException(I18n.get("extension.missingStore", name)); + throw new ValidationException(I18n.get("extension.missingStore", name)); } } - public static void hostFeature(ShellStore host, Predicate predicate, String name) { + public static void hostFeature(ShellStore host, Predicate predicate, String name) throws ValidationException { if (!predicate.test(host)) { - throw new IllegalArgumentException(I18n.get("extension.hostFeatureUnsupported", name)); + throw new ValidationException(I18n.get("extension.hostFeatureUnsupported", name)); } } } diff --git a/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties b/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties index 4cb7ac80..58fe26ee 100644 --- a/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties +++ b/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties @@ -5,7 +5,7 @@ lf=LF (Linux) none=None common=Common other=Other -nullPointer=Null Pointer: $MSG$ +nullPointer=Null Pointer mustNotBeEmpty=$NAME$ must not be empty null=$VALUE$ must be not null hostFeatureUnsupported=Host does not support the feature $FEATURE$