From 5e8dd42dd95e5ac4920393f573560f0d7343a13a Mon Sep 17 00:00:00 2001 From: Christopher Schnick Date: Wed, 23 Nov 2022 14:15:21 +0100 Subject: [PATCH] More fixes for file stores --- .../xpipe/api/connector/XPipeConnection.java | 2 +- .../exchange/QueryDataSourceExchange.java | 1 - .../core/store/CommandProcessControl.java | 50 +++++++++++++------ .../java/io/xpipe/core/store/FileStore.java | 15 ++++++ .../java/io/xpipe/core/store/LocalStore.java | 3 +- .../io/xpipe/core/store/MachineStore.java | 23 ++++----- .../io/xpipe/core/store/ProcessControl.java | 10 ++++ .../xpipe/core/store/ShellProcessControl.java | 10 ++++ .../java/io/xpipe/core/store/ShellType.java | 6 +++ .../java/io/xpipe/core/store/ShellTypes.java | 46 +++++++++++------ .../main/java/io/xpipe/core/util/OsType.java | 17 +++++++ .../extension/util/DaemonExtensionTest.java | 20 +++++++- .../xpipe/extension/util/ExtensionTest.java | 23 --------- .../extension/util/LocalExtensionTest.java | 10 ++++ 14 files changed, 165 insertions(+), 71 deletions(-) diff --git a/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java b/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java index 8a8d67a8..bc021555 100644 --- a/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java +++ b/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java @@ -64,7 +64,7 @@ public final class XPipeConnection extends BeaconConnection { public static Optional waitForStartup(Process process) { for (int i = 0; i < 160; i++) { - if (process != null && !process.isAlive()) { + if (process != null && !process.isAlive() && process.exitValue() != 0) { return Optional.empty(); } diff --git a/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java b/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java index 0ead9871..1ad41b22 100644 --- a/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java +++ b/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java @@ -38,7 +38,6 @@ public class QueryDataSourceExchange implements MessageExchange { @NonNull DataSourceId id; - @NonNull String information; @NonNull diff --git a/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java b/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java index 1bbaa986..b19d36a2 100644 --- a/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java +++ b/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java @@ -2,34 +2,48 @@ package io.xpipe.core.store; import java.io.*; import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; public interface CommandProcessControl extends ProcessControl { default InputStream startExternalStdout() throws Exception { - start(); - discardErr(); - return new FilterInputStream(getStdout()) { - @Override - public void close() throws IOException { - getStdout().close(); - CommandProcessControl.this.close(); - } - }; + try { + start(); + + AtomicReference err = new AtomicReference<>(""); + accumulateStderr(s -> err.set(s)); + + return new FilterInputStream(getStdout()) { + @Override + public void close() throws IOException { + CommandProcessControl.this.close(); + if (!err.get().isEmpty()) { + throw new IOException(err.get()); + } + } + }; + } catch (Exception ex) { + close(); + throw ex; + } } default OutputStream startExternalStdin() throws Exception { - try (CommandProcessControl pc = start()) { - pc.discardOut(); - pc.discardErr(); + try { + start(); + discardOut(); + discardErr(); return new FilterOutputStream(getStdin()) { @Override public void close() throws IOException { - pc.getStdin().close(); - pc.close(); + closeStdin(); + CommandProcessControl.this.close(); } }; - } catch (Exception e) { - throw e; + } catch (Exception ex) { + close(); + throw ex; } } @@ -53,6 +67,10 @@ public interface CommandProcessControl extends ProcessControl { readOrThrow(); } + void accumulateStdout(Consumer con); + + void accumulateStderr(Consumer con); + public String readOrThrow() throws Exception; public default boolean startAndCheckExit() { diff --git a/core/src/main/java/io/xpipe/core/store/FileStore.java b/core/src/main/java/io/xpipe/core/store/FileStore.java index 6200dffb..2f51fd17 100644 --- a/core/src/main/java/io/xpipe/core/store/FileStore.java +++ b/core/src/main/java/io/xpipe/core/store/FileStore.java @@ -6,9 +6,11 @@ import lombok.Getter; import lombok.experimental.SuperBuilder; import lombok.extern.jackson.Jacksonized; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; +import java.util.regex.Pattern; /** * Represents a file located on a file system. @@ -27,6 +29,15 @@ public class FileStore extends JacksonizedValue implements FilenameStore, Stream this.file = file; } + public String getParent() { + var matcher = Pattern.compile("^(.+?)[^\\\\/]+$").matcher(file); + if (!matcher.matches()) { + throw new IllegalArgumentException("Unable to determine parent of " + file); + } + + return matcher.group(1); + } + public final boolean isLocal() { return fileSystem instanceof LocalStore; } @@ -59,6 +70,10 @@ public class FileStore extends JacksonizedValue implements FilenameStore, Stream @Override public OutputStream openOutput() throws Exception { + if (!fileSystem.mkdirs(getParent())) { + throw new IOException("Unable to create directory: " + getParent()); + } + return fileSystem.openOutput(file); } diff --git a/core/src/main/java/io/xpipe/core/store/LocalStore.java b/core/src/main/java/io/xpipe/core/store/LocalStore.java index a2b73c4f..27baa5ff 100644 --- a/core/src/main/java/io/xpipe/core/store/LocalStore.java +++ b/core/src/main/java/io/xpipe/core/store/LocalStore.java @@ -25,7 +25,7 @@ public class LocalStore extends JacksonizedValue implements FileSystemStore, Mac @Override public boolean mkdirs(String file) throws Exception { try { - Files.createDirectories(Path.of(file).getParent()); + Files.createDirectories(Path.of(file)); return true; } catch (Exception ex) { return false; @@ -40,7 +40,6 @@ public class LocalStore extends JacksonizedValue implements FileSystemStore, Mac @Override public OutputStream openOutput(String file) throws Exception { - mkdirs(file); var p = Path.of(file); return Files.newOutputStream(p); } diff --git a/core/src/main/java/io/xpipe/core/store/MachineStore.java b/core/src/main/java/io/xpipe/core/store/MachineStore.java index 756ed1c1..073511c2 100644 --- a/core/src/main/java/io/xpipe/core/store/MachineStore.java +++ b/core/src/main/java/io/xpipe/core/store/MachineStore.java @@ -7,8 +7,7 @@ public interface MachineStore extends FileSystemStore, ShellStore { @Override default void validate() throws Exception { - try (ShellProcessControl pc = create().start()) { - } + try (ShellProcessControl pc = create().start()) {} } public default boolean isLocal() { @@ -24,29 +23,29 @@ public interface MachineStore extends FileSystemStore, ShellStore { @Override public default InputStream openInput(String file) throws Exception { - return create().commandListFunction(proc -> proc.getShellType().createFileReadCommand(file)) + return create().commandListFunction(proc -> proc.getShellType().createFileReadCommand(proc.getOsType().normalizeFileName(file))) .startExternalStdout(); } @Override public default OutputStream openOutput(String file) throws Exception { - return create().commandListFunction(proc -> proc.getShellType().createFileWriteCommand(file)) + return create().commandListFunction(proc -> proc.getShellType().createFileWriteCommand(proc.getOsType().normalizeFileName(file))) .startExternalStdin(); } @Override public default boolean exists(String file) throws Exception { - var r = create().commandListFunction(proc -> proc.getShellType().createFileExistsCommand(file)) - .start() - .discardAndCheckExit(); - return r; + try (var pc = create().commandListFunction(proc -> proc.getShellType().createFileExistsCommand(proc.getOsType().normalizeFileName(file))) + .start()) { + return pc.discardAndCheckExit(); + } } @Override public default boolean mkdirs(String file) throws Exception { - var r = create().commandListFunction(proc -> proc.getShellType().createMkdirsCommand(file)) - .start() - .discardAndCheckExit(); - return r; + try (var pc = create().commandListFunction(proc -> proc.getShellType().createMkdirsCommand(proc.getOsType().normalizeFileName(file))) + .start()) { + return pc.discardAndCheckExit(); + } } } diff --git a/core/src/main/java/io/xpipe/core/store/ProcessControl.java b/core/src/main/java/io/xpipe/core/store/ProcessControl.java index f795e815..0b711760 100644 --- a/core/src/main/java/io/xpipe/core/store/ProcessControl.java +++ b/core/src/main/java/io/xpipe/core/store/ProcessControl.java @@ -4,9 +4,19 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.Charset; +import java.util.List; +import java.util.stream.Collectors; public interface ProcessControl extends AutoCloseable { + static String join(List command) { + return command.stream().map(s -> s.contains(" ") ? "\"" + s + "\"" : s).collect(Collectors.joining(" ")); + } + + void closeStdin() throws IOException; + + boolean isStdinClosed(); + boolean isRunning(); ShellType getShellType(); diff --git a/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java b/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java index c86709d6..4e588de6 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java +++ b/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java @@ -12,6 +12,16 @@ import java.util.stream.Collectors; public interface ShellProcessControl extends ProcessControl { + default String executeSimpleCommand(String command) throws Exception { + try (CommandProcessControl c = command(command).start()) { + return c.readOrThrow(); + } + } + + default String executeSimpleCommand(ShellType type, String command) throws Exception { + return executeSimpleCommand(type.switchTo(command)); + } + int getProcessId(); OsType getOsType(); diff --git a/core/src/main/java/io/xpipe/core/store/ShellType.java b/core/src/main/java/io/xpipe/core/store/ShellType.java index 50d7cad6..681cbd16 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellType.java +++ b/core/src/main/java/io/xpipe/core/store/ShellType.java @@ -9,6 +9,10 @@ import java.util.List; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") public interface ShellType { + default String joinCommands(String... s) { + return String.join(getConcatenationOperator(), s); + } + void elevate(ShellProcessControl control, String command, String displayCommand) throws Exception; default String getExitCommand() { @@ -29,6 +33,8 @@ public interface ShellType { String queryShellProcessId(ShellProcessControl control) throws Exception; + String getSetVariableCommand(String variableName, String value); + List openCommand(); String switchTo(String cmd); diff --git a/core/src/main/java/io/xpipe/core/store/ShellTypes.java b/core/src/main/java/io/xpipe/core/store/ShellTypes.java index 32f6350d..6d0dfad6 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellTypes.java +++ b/core/src/main/java/io/xpipe/core/store/ShellTypes.java @@ -46,6 +46,11 @@ public class ShellTypes { @Value public static class Cmd implements ShellType { + @Override + public String getSetVariableCommand(String variableName, String value) { + return "set \"" + variableName + "=" + value + "\""; + } + @Override public String getEchoCommand(String s, boolean toErrorStream) { return toErrorStream ? "(echo " + s + ")1>&2" : "echo " + s; @@ -103,7 +108,7 @@ public class ShellTypes { @Override public List createMkdirsCommand(String dirs) { - return List.of("lmkdir", dirs); + return List.of("mkdir", dirs); } @Override @@ -113,12 +118,12 @@ public class ShellTypes { @Override public List createFileWriteCommand(String file) { - return List.of("Out-File", "-FilePath", file); + return List.of("findstr", "\"^\"", ">", file); } @Override public List createFileExistsCommand(String file) { - return List.of("if", "exist", file, "echo", "hi"); + return List.of("dir", "/a", file); } @Override @@ -158,6 +163,11 @@ public class ShellTypes { @Value public static class PowerShell implements ShellType { + @Override + public String getSetVariableCommand(String variableName, String value) { + return "set " + variableName + "=" + value; + } + @Override public String queryShellProcessId(ShellProcessControl control) throws IOException { control.writeLine("powershell (Get-WmiObject Win32_Process -Filter ProcessId=$PID).ParentProcessId"); @@ -217,24 +227,24 @@ public class ShellTypes { return "powershell.exe -Command " + cmd; } - @Override - public List createMkdirsCommand(String dirs) { - return List.of("New-Item", "-Path", dirs, "-ItemType", "Directory"); - } - @Override public List createFileReadCommand(String file) { - return List.of("Get-Content", file); + return List.of("cmd", "/c", "type", file); } @Override public List createFileWriteCommand(String file) { - return List.of("Out-File", "-FilePath", file); + return List.of("cmd", "/c", "findstr", "\"^\"", ">", file); + } + + @Override + public List createMkdirsCommand(String dirs) { + return List.of("cmd", "/c", "mkdir", dirs); } @Override public List createFileExistsCommand(String file) { - return List.of("Test-Path", "-path", file); + return List.of("cmd", "/c", "dir", "/a", file); } @Override @@ -285,12 +295,12 @@ public class ShellTypes { @Override public void elevate(ShellProcessControl control, String command, String displayCommand) throws Exception { if (control.getElevationPassword() == null) { - control.executeCommand("SUDO_ASKPASS=/bin/false; sudo -p \"\" -S " + command); + control.executeCommand("SUDO_ASKPASS=/bin/false sudo -p \"\" -S " + command); return; } - control.executeCommand("sudo -p \"\" -S " + command); - // Thread.sleep(200); + // For sudo to always query for a password by using the -k switch + control.executeCommand("sudo -p \"\" -k -S " + command); control.writeLine(control.getElevationPassword().getSecretValue()); } @@ -314,6 +324,12 @@ public class ShellTypes { return matcher.group(0); } } + + @Override + public String getSetVariableCommand(String variableName, String value) { + return variableName + "=" + value; + } + @Override public List openCommand() { return List.of("sh", "-i", "-l"); @@ -336,7 +352,7 @@ public class ShellTypes { @Override public List createFileWriteCommand(String file) { - return List.of(file); + return List.of("cat", ">", file); } @Override diff --git a/core/src/main/java/io/xpipe/core/util/OsType.java b/core/src/main/java/io/xpipe/core/util/OsType.java index 830f8fbe..4475a0d1 100644 --- a/core/src/main/java/io/xpipe/core/util/OsType.java +++ b/core/src/main/java/io/xpipe/core/util/OsType.java @@ -18,6 +18,8 @@ public interface OsType { String getName(); + String normalizeFileName(String file); + Map getProperties(ShellProcessControl pc) throws Exception; String determineOperatingSystemName(ShellProcessControl pc) throws Exception; @@ -46,6 +48,11 @@ public interface OsType { return "Windows"; } + @Override + public String normalizeFileName(String file) { + return String.join("\\", file.split("[\\\\/]+")); + } + @Override public Map getProperties(ShellProcessControl pc) throws Exception { try (CommandProcessControl c = @@ -79,6 +86,11 @@ public interface OsType { static class Linux implements OsType { + @Override + public String normalizeFileName(String file) { + return String.join("/", file.split("[\\\\/]+")); + } + @Override public String getName() { return "Linux"; @@ -139,6 +151,11 @@ public interface OsType { static class Mac implements OsType { + @Override + public String normalizeFileName(String file) { + return String.join("/", file.split("[\\\\/]+")); + } + @Override public String getName() { return "Mac"; diff --git a/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java index 5152670b..bebc9af5 100644 --- a/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java @@ -1,13 +1,31 @@ package io.xpipe.extension.util; +import io.xpipe.api.DataSource; import io.xpipe.api.util.XPipeDaemonController; +import io.xpipe.core.store.DataStore; +import io.xpipe.core.util.JacksonMapper; +import io.xpipe.extension.XPipeServiceProviders; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -public class DaemonExtensionTest { +public class DaemonExtensionTest extends ExtensionTest { + + public static DataSource getSource(String type, DataStore store) { + return DataSource.create(null, type, store); + } + + public static DataSource getSource(String type, String file) { + return DataSource.create(null, type, getResource(file)); + } + + public static DataSource getSource(io.xpipe.core.source.DataSource source) { + return DataSource.create(null, source); + } @BeforeAll public static void setup() throws Exception { + JacksonMapper.initModularized(ModuleLayer.boot()); + XPipeServiceProviders.load(ModuleLayer.boot()); XPipeDaemonController.start(); } diff --git a/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java index a4959d8a..bf3c5c06 100644 --- a/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java @@ -1,18 +1,13 @@ package io.xpipe.extension.util; -import io.xpipe.api.DataSource; import io.xpipe.core.store.DataStore; import io.xpipe.core.store.FileStore; -import io.xpipe.core.util.JacksonMapper; -import io.xpipe.extension.XPipeServiceProviders; import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; import java.nio.file.Path; public class ExtensionTest { - @SneakyThrows public static DataStore getResource(String name) { var url = DaemonExtensionTest.class.getClassLoader().getResource(name); @@ -22,22 +17,4 @@ public class ExtensionTest { var file = Path.of(url.toURI()).toString(); return FileStore.local(Path.of(file)); } - - public static DataSource getSource(String type, DataStore store) { - return DataSource.create(null, type, store); - } - - public static DataSource getSource(String type, String file) { - return DataSource.create(null, type, getResource(file)); - } - - public static DataSource getSource(io.xpipe.core.source.DataSource source) { - return DataSource.create(null, source); - } - - @BeforeAll - public static void setup() throws Exception { - JacksonMapper.initModularized(ModuleLayer.boot()); - XPipeServiceProviders.load(ModuleLayer.boot()); - } } diff --git a/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java index 5e29f115..0a1b43ab 100644 --- a/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java @@ -1,4 +1,14 @@ package io.xpipe.extension.util; +import io.xpipe.core.util.JacksonMapper; +import io.xpipe.extension.XPipeServiceProviders; +import org.junit.jupiter.api.BeforeAll; + public class LocalExtensionTest extends ExtensionTest { + + @BeforeAll + public static void setup() throws Exception { + JacksonMapper.initModularized(ModuleLayer.boot()); + XPipeServiceProviders.load(ModuleLayer.boot()); + } }