diff --git a/build.gradle b/build.gradle index 0d4d3d65..e4724d15 100644 --- a/build.gradle +++ b/build.gradle @@ -27,4 +27,5 @@ project.ext { privateExtensions = file("$rootDir/private_extensions.txt").exists() ? file("$rootDir/private_extensions.txt").readLines() : [] isFullRelease = System.getenv('RELEASE') != null && Boolean.parseBoolean(System.getenv('RELEASE')) versionString = file('version').text + (isFullRelease ? '' : '-SNAPSHOT') + canonicalVersionString = file('version').text } diff --git a/dist/jpackage.gradle b/dist/jpackage.gradle index 4947e803..76bf3985 100644 --- a/dist/jpackage.gradle +++ b/dist/jpackage.gradle @@ -55,10 +55,10 @@ jlink { imageName = 'xpiped' if (org.gradle.internal.os.OperatingSystem.current().isWindows()) { icon = "logo/logo.ico" - appVersion = version + appVersion = rootProject.canonicalVersionString } else if (org.gradle.internal.os.OperatingSystem.current().isLinux()) { icon = "logo/logo.png" - appVersion = version + appVersion = rootProject.canonicalVersionString } else { icon = "logo/logo.icns" @@ -80,7 +80,7 @@ jlink { resourceDir = file("$projectDir/misc/mac") // Mac does not like a zero major version - def modifiedVersion = version.toString() + def modifiedVersion = rootProject.canonicalVersionString if (Integer.parseInt(modifiedVersion.substring(0, 1)) == 0) { modifiedVersion = "1" + modifiedVersion.substring(1) } diff --git a/ext/jdbcx/build.gradle b/ext/jdbcx/build.gradle index fce335e8..d0d01af8 100644 --- a/ext/jdbcx/build.gradle +++ b/ext/jdbcx/build.gradle @@ -1 +1,34 @@ -plugins { id 'java' } +plugins { + id 'java' + id "org.moditect.gradleplugin" version "1.0.0-rc3" +} + +apply from: "$rootDir/gradle/gradle_scripts/lombok.gradle" +apply from: "$rootDir/gradle/gradle_scripts/extension.gradle" + +compileJava { + doFirst { + options.compilerArgs += [ + '--module-path', classpath.asPath + ] + classpath = files() + } +} + +jar.destinationDirectory = project(':jdbc').jar.destinationDirectory + +configurations { + compileOnly.extendsFrom(dep) + testImplementation.extendsFrom(dep) +} + +dependencies { + compileOnly project(':app') + compileOnly project(':jdbc') + compileOnly 'net.synedra:validatorfx:0.3.1' + implementation 'com.microsoft.sqlserver:mssql-jdbc:11.2.1.jre17' + implementation 'org.rauschig:jarchivelib:1.2.0' + testImplementation project(':base') + testCompileOnly project(':app') +} + diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/JdbcxJacksonModule.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/JdbcxJacksonModule.java new file mode 100644 index 00000000..0502dc15 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/JdbcxJacksonModule.java @@ -0,0 +1,14 @@ +package io.xpipe.ext.jdbcx; + +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import io.xpipe.ext.jdbcx.mssql.MssqlAddress; + +public class JdbcxJacksonModule extends SimpleModule { + + @Override + public void setupModule(SetupContext context) { + context.registerSubtypes( + new NamedType(MssqlAddress.class)); + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlAddress.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlAddress.java new file mode 100644 index 00000000..fa5eefa4 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlAddress.java @@ -0,0 +1,21 @@ +package io.xpipe.ext.jdbcx.mssql; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.ext.jdbc.address.JdbcBasicAddress; +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +@JsonTypeName("mssqlInstance") +@SuperBuilder +@Jacksonized +@Getter +public class MssqlAddress extends JdbcBasicAddress { + + String instance; + + @Override + public String toAddressString() { + return getHostname() + (instance != null ? "\\" + instance : "") + (getPort() != null ? ":" + getPort() : ""); + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlDialect.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlDialect.java new file mode 100644 index 00000000..fb2e69c4 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlDialect.java @@ -0,0 +1,301 @@ +package io.xpipe.ext.jdbcx.mssql; + +import com.microsoft.sqlserver.jdbc.Geography; +import com.microsoft.sqlserver.jdbc.Geometry; +import com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement; +import com.microsoft.sqlserver.jdbc.SQLServerResultSet; +import io.xpipe.core.charsetter.Charsetter; +import io.xpipe.core.data.node.DataStructureNode; +import io.xpipe.core.data.node.TupleNode; +import io.xpipe.core.data.node.ValueNode; +import io.xpipe.ext.jdbc.JdbcDataTypeCategory; +import io.xpipe.ext.jdbc.JdbcDialect; +import io.xpipe.ext.jdbc.JdbcHelper; +import io.xpipe.ext.jdbc.source.JdbcTableParameterMap; +import microsoft.sql.Types; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class MssqlDialect implements JdbcDialect { + + @Override + public boolean matches(Connection connection) throws SQLException { + return connection.getMetaData().getDatabaseProductName().equals("Microsoft SQL Server"); + } + + private static final List ADDITIONAL_CATEGORIES = List.of( + new JdbcDataTypeCategory.UserDefinedCategory(Types.GEOMETRY) { + @Override + public ValueNode getValue(ResultSet result, int jdbcDataType, int index) throws SQLException { + SQLServerResultSet sqlServerResultSet = (SQLServerResultSet) result; + var geometry = sqlServerResultSet.getGeometry(index); + return geometry != null ? ValueNode.of(geometry.STAsText()) : ValueNode.nullValue(); + } + + @Override + public void setValueNonNull( + PreparedStatement statement, int jdbcDataType, int index, DataStructureNode value) + throws SQLException { + SQLServerPreparedStatement p = (SQLServerPreparedStatement) statement; + Geometry geometry = Geometry.parse(value.asString()); + p.setGeometry(index, geometry); + } + }, + new JdbcDataTypeCategory.UserDefinedCategory(Types.GEOGRAPHY) { + @Override + public ValueNode getValue(ResultSet result, int jdbcDataType, int index) throws SQLException { + SQLServerResultSet sqlServerResultSet = (SQLServerResultSet) result; + var geography = sqlServerResultSet.getGeography(index); + return geography != null ? ValueNode.of(geography.STAsText()) : ValueNode.nullValue(); + } + + @Override + public void setValueNonNull( + PreparedStatement statement, int jdbcDataType, int index, DataStructureNode value) + throws SQLException { + SQLServerPreparedStatement p = (SQLServerPreparedStatement) statement; + Geography geography = Geography.parse(value.asString()); + p.setGeography(index, geography); + } + }); + + @Override + public List getAdditionalCategories() { + return ADDITIONAL_CATEGORIES; + } + + @Override + public String createTableLikeSql(String newTable, String oldTable, List columns, List identifiers) { + var select = columns != null ? String.join(",", columns) : "*"; + return String.format( + """ + SELECT %s + into %s + FROM %s + where 0=1 + union all + SELECT %s + FROM %s + where 0=1""", + select, newTable, oldTable, select, oldTable); + } + + @Override + public PreparedStatement createTableMergeStatement( + Connection connection, String source, String target, JdbcTableParameterMap parameterMap) + throws SQLException { + var memoryOptimized = "1" + .equals(JdbcHelper.executeSingletonQueryStatement( + connection, + String.format("SELECT OBJECTPROPERTY(OBJECT_ID('%s'),'TableIsMemoryOptimized')", target))); + + var equalJoinCheck = parameterMap.getInformation().getIdentifiers().stream() + .map(s -> "T." + s + " = " + "S." + s) + .collect(Collectors.joining(",")); + var insert = String.join(",", parameterMap.getInformation().getUpdateTableColumns()); + var columnList = parameterMap.getInformation().getUpdateTableColumns().stream() + .map(s -> "S." + s) + .collect(Collectors.joining(",")); + var update = parameterMap.getInformation().getUpdateTableColumns().stream() + .map(s -> "T." + s + " = " + "S." + s) + .collect(Collectors.joining(",")); + + if (memoryOptimized) { + var unequalJoinedCheck = parameterMap.getInformation().getIdentifiers().stream() + .map(s -> "T." + s + " <> " + "S." + s) + .collect(Collectors.joining(",")); + var query = String.format( + """ + UPDATE T + SET %s + FROM %s AS S, %s AS T + WHERE %s + + INSERT INTO %s (%s) + SELECT %s + FROM %s S INNER JOIN %s T + ON %s""", + update, + source, + target, + equalJoinCheck, + target, + insert, + columnList, + source, + target, + unequalJoinedCheck); + return connection.prepareStatement(query); + } + + var query = String.format( + """ + MERGE %s AS T + USING %s AS S + ON %s + WHEN NOT MATCHED BY TARGET THEN + INSERT (%s) + VALUES (%s) + WHEN MATCHED THEN UPDATE SET + %s + WHEN NOT MATCHED BY SOURCE THEN + DELETE;""", + target, source, equalJoinCheck, insert, columnList, update); + return connection.prepareStatement(query); + } + + @Override + public PreparedStatement createUpsertStatement(Connection connection, String table, JdbcTableParameterMap map) + throws SQLException { + var values = map.getInsertTableColumns().stream().map(s -> "?").collect(Collectors.joining(",")); + var equalJoinCheck = map.hasIdentifiers() + ? map.getInformation().getIdentifiers().stream() + .map(s -> "" + s + " = " + "?") + .collect(Collectors.joining(",")) + : "0=1"; + var insert = String.join(",", map.getInsertTableColumns()); + var update = map.getInformation().getUpdateTableColumns().stream() + .map(s -> "" + s + " = " + "?") + .collect(Collectors.joining(",")); + + var insertStatement = String.format("INSERT INTO %s (%s)\nVALUES (%s)", table, insert, values); + var upsertStatement = String.format( + """ + UPDATE %s + SET %s + WHERE %s + IF @@ROWCOUNT = 0 + %s""", + table, update, equalJoinCheck, insertStatement); + + if (map.isCanPerformUpdates()) { + return connection.prepareStatement(upsertStatement); + } else { + return connection.prepareStatement(insertStatement); + } + } + + @Override + public void disableConstraints(Connection connection) throws SQLException { + JdbcHelper.execute(connection, "EXEC sp_MSforeachtable \"ALTER TABLE ? NOCHECK CONSTRAINT ALL\""); + } + + @Override + public void enableConstraints(Connection connection) throws SQLException { + JdbcHelper.execute(connection, "EXEC sp_MSforeachtable \"ALTER TABLE ? WITH CHECK CHECK CONSTRAINT ALL\""); + } + + @Override + public PreparedStatement fillUpsertStatement( + PreparedStatement statement, + TupleNode tuple, + JdbcTableParameterMap parameterMap, + Charsetter.FailableConsumer filler) + throws SQLException { + if (parameterMap.isCanPerformUpdates()) { + for (int i = 0; i < tuple.getNodes().size(); i++) { + if (parameterMap.map(i).isEmpty() + || !parameterMap + .getInformation() + .getUpdateTableColumns() + .contains(parameterMap + .getInformation() + .getAllTableColumns() + .get(parameterMap.map(i).getAsInt()))) { + continue; + } + + filler.accept(i); + } + + for (String primaryKey : parameterMap.getInformation().getIdentifiers()) { + var tupleIndex = parameterMap.getTupleIndexOfColumnName(primaryKey); + filler.accept(tupleIndex); + } + } + + for (int i = 0; i < tuple.getNodes().size(); i++) { + if (parameterMap.map(i).isEmpty() + || !parameterMap + .getInsertTableColumns() + .contains(parameterMap + .getInformation() + .getAllTableColumns() + .get(parameterMap.map(i).getAsInt()))) { + continue; + } + + filler.accept(i); + } + + return statement; + } + + @Override + public List determineStandardTables(Connection connection, List tables) throws SQLException { + var alteredTables = new ArrayList<>(tables); + try (PreparedStatement statement = connection.prepareStatement( + """ + SELECT + OBJECT_SCHEMA_NAME(object_id) AS 'Table Schema', + OBJECT_NAME(object_id) AS 'Temporal Table', + OBJECT_NAME(history_table_id) AS 'History Table' + FROM sys.tables + WHERE temporal_type = 2""")) { + var result = JdbcHelper.executeQueryStatement(statement); + while (result.next()) { + var schema = result.getString(1); + var temporal = schema + "." + result.getString(3); + alteredTables.remove(temporal); + } + } + + alteredTables.removeIf(s -> s.startsWith("sys.")); + return alteredTables; + } + + @Override + public List determineAdditionalGeneratedColumns(Connection connection, String table, List columns) + throws SQLException { + var alwaysGenerated = getColumnProperty(connection, table, columns, "GeneratedAlwaysType"); + var isIdentity = getColumnProperty(connection, table, columns, "IsIdentity"); + + var list = new ArrayList(); + + for (int i = 0; i < columns.size(); i++) { + var remove = false; + if (!"0".equals(alwaysGenerated.get(i))) { + remove = true; + } + if (!"0".equals(isIdentity.get(i))) { + remove = true; + } + if (remove) { + list.add(columns.get(i)); + } + } + return list; + } + + public List getColumnProperty(Connection connection, String table, List columns, String name) + throws SQLException { + PreparedStatement s = connection.prepareStatement(String.format( + """ + SELECT COLUMNPROPERTY(id, name, '%s') + FROM sys.syscolumns + WHERE id=OBJECT_ID('%s') + ORDER BY colid""", + name, table)); + var list = new ArrayList(); + try (ResultSet resultSet = JdbcHelper.executeQueryStatement(s)) { + list.addAll(JdbcHelper.readSingleColumnResultSet(resultSet)); + } + return list; + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlSimpleStore.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlSimpleStore.java new file mode 100644 index 00000000..ca5c7b8a --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlSimpleStore.java @@ -0,0 +1,50 @@ +package io.xpipe.ext.jdbcx.mssql; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.store.ShellStore; +import io.xpipe.ext.jdbc.JdbcDatabaseServerStore; +import io.xpipe.ext.jdbc.address.JdbcAddress; +import io.xpipe.ext.jdbc.auth.AuthMethod; +import io.xpipe.ext.jdbc.auth.SimpleAuthMethod; +import io.xpipe.ext.jdbc.auth.WindowsAuth; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +import java.util.HashMap; +import java.util.Map; + +@JsonTypeName("mssqlSimple") +@SuperBuilder +@Jacksonized +public class MssqlSimpleStore extends JdbcDatabaseServerStore implements MssqlStore { + + public MssqlSimpleStore(ShellStore proxy, JdbcAddress address, AuthMethod auth) { + super(proxy, address, auth); + } + + @Override + public String toUrl() { + var base = + "jdbc:sqlserver://" + address.toAddressString() + ";encrypt=false;" + "trustServerCertificate=false;"; + if (auth instanceof WindowsAuth) { + base = base + "integratedSecurity=true;"; + } + return base; + } + + @Override + public Map createProperties() { + var p = new HashMap(); + + switch (auth) { + case SimpleAuthMethod s -> { + p.put("user", s.getUsername()); + p.put("password", s.getPassword().getSecretValue()); + } + case WindowsAuth a -> {} + default -> {} + } + + return p; + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlStore.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlStore.java new file mode 100644 index 00000000..f1b51b61 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlStore.java @@ -0,0 +1,13 @@ +package io.xpipe.ext.jdbcx.mssql; + +import io.xpipe.ext.jdbc.JdbcBaseStore; + +import java.util.Map; + +public interface MssqlStore extends JdbcBaseStore { + + @Override + default Map createDefaultProperties() { + return Map.of("applicationName", "X-Pipe", "loginTimeout", "5"); + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlStoreProvider.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlStoreProvider.java new file mode 100644 index 00000000..eac98134 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlStoreProvider.java @@ -0,0 +1,201 @@ +package io.xpipe.ext.jdbcx.mssql; + +import io.xpipe.core.store.DataStore; +import io.xpipe.ext.jdbc.JdbcGuiHelper; +import io.xpipe.ext.jdbc.JdbcStoreProvider; +import io.xpipe.ext.jdbc.auth.AuthMethod; +import io.xpipe.ext.jdbc.auth.SimpleAuthMethod; +import io.xpipe.ext.jdbc.auth.WindowsAuth; +import io.xpipe.extension.GuiDialog; +import io.xpipe.extension.I18n; +import io.xpipe.extension.fxcomps.Comp; +import io.xpipe.extension.fxcomps.impl.ChoicePaneComp; +import io.xpipe.extension.fxcomps.impl.TabPaneComp; +import io.xpipe.extension.fxcomps.impl.VerticalComp; +import io.xpipe.extension.util.*; +import javafx.beans.binding.Bindings; +import javafx.beans.property.Property; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.value.ObservableValue; +import javafx.scene.layout.Region; + +import java.util.List; +import java.util.Map; + +public class MssqlStoreProvider extends JdbcStoreProvider { + + public static final String PROTOCOL = "sqlserver"; + public static final int DEFAULT_PORT = 1433; + public static final String DEFAULT_USERNAME = "sa"; + + public MssqlStoreProvider() { + super("com.microsoft.sqlserver.jdbc.SQLServerDriver"); + } + + @Override + public GuiDialog guiDialog(Property store) { + var wizValue = new SimpleObjectProperty( + store.getValue() instanceof MssqlSimpleStore ? store.getValue() : defaultStore()); + var wizardDialog = wizard(wizValue); + var wizard = new TabPaneComp.Entry(I18n.observable("jdbc.connectionWizard"), null, wizardDialog.getComp()); + + var urlVal = new SimpleValidator(); + var urlValue = new SimpleObjectProperty<>(store.getValue() instanceof MssqlUrlStore ? store.getValue() : null); + var url = new TabPaneComp.Entry(I18n.observable("jdbc.connectionUrl"), null, url(urlValue, urlVal)); + + var stringVal = new SimpleValidator(); + var stringValue = new SimpleObjectProperty<>(store.getValue()); + var string = + new TabPaneComp.Entry(I18n.observable("jdbc.connectionString"), null, string(stringValue, stringVal)); + + var selected = new SimpleObjectProperty<>(store.getValue() instanceof MssqlUrlStore ? url : wizard); + + var map = Map.of( + wizard, wizardDialog.getValidator(), + url, urlVal, + string, stringVal); + var orVal = new ExclusiveValidator<>(map, selected); + + var propMap = Map.of( + wizard, wizValue, + url, urlValue, + string, stringValue); + PropertiesHelper.bindExclusive(selected, propMap, store); + + var pane = new TabPaneComp(selected, List.of(wizard, url)); + return new GuiDialog(pane, orVal); + } + + private Comp string(Property store, Validator val) { + return Comp.of(() -> new Region()); + } + + private Comp url(Property store, Validator val) { + return JdbcGuiHelper.url(PROTOCOL, MssqlUrlStore.class, store, val); + } + + private GuiDialog wizard(Property store) { + MssqlSimpleStore st = (MssqlSimpleStore) store.getValue(); + Property addrProp = + new SimpleObjectProperty<>(st != null ? (MssqlAddress) st.getAddress() : null); + + var host = new SimpleStringProperty( + addrProp.getValue() != null ? addrProp.getValue().getHostname() : null); + var port = new SimpleObjectProperty<>( + addrProp.getValue() != null ? addrProp.getValue().getPort() : null); + var instance = new SimpleStringProperty( + addrProp.getValue() != null ? addrProp.getValue().getInstance() : null); + var addressValidator = new SimpleValidator(); + var addrQ = new DynamicOptionsBuilder(I18n.observable("jdbc.connection")) + .addString(I18n.observable("jdbc.host"), host) + .nonNull(addressValidator) + .addInteger(I18n.observable("jdbc.port"), port) + .addString(I18n.observable("jdbc.instance"), instance) + .bind( + () -> { + return MssqlAddress.builder() + .hostname(host.get()) + .port(port.get()) + .instance(instance.get()) + .build(); + }, + addrProp) + .buildComp(); + + Property authProp = new SimpleObjectProperty<>(st != null ? st.getAuth() : null); + Property passwordAuthProp = new SimpleObjectProperty<>( + authProp.getValue() instanceof SimpleAuthMethod ? (SimpleAuthMethod) authProp.getValue() : null); + var passwordAuthenticationValidator = new SimpleValidator(); + var passwordAuthQ = Comp.of(() -> { + var user = new SimpleStringProperty( + passwordAuthProp.getValue() != null + ? passwordAuthProp.getValue().getUsername() + : DEFAULT_USERNAME); + var pass = new SimpleObjectProperty<>( + passwordAuthProp.getValue() != null + ? passwordAuthProp.getValue().getPassword() + : null); + return new DynamicOptionsBuilder(false) + .addString(I18n.observable("jdbc.username"), user) + .nonNull(passwordAuthenticationValidator) + .addSecret(I18n.observable("jdbc.password"), pass) + .nonNull(passwordAuthenticationValidator) + .bind( + () -> { + return new SimpleAuthMethod(user.get(), pass.get()); + }, + passwordAuthProp) + .build(); + }); + + var passwordEntry = new ChoicePaneComp.Entry(I18n.observable("jdbc.passwordAuth"), passwordAuthQ); + var windowsAuthenticationValidator = new SimpleValidator(); + var windowsEntry = new ChoicePaneComp.Entry(I18n.observable("jdbc.windowsAuth"), Comp.of(Region::new)); + var entries = List.of(passwordEntry, windowsEntry); + var authSelected = new SimpleObjectProperty( + authProp.getValue() == null || authProp.getValue() instanceof SimpleAuthMethod + ? passwordEntry + : windowsEntry); + var map = Map.of( + passwordEntry, passwordAuthenticationValidator, + windowsEntry, windowsAuthenticationValidator); + var authenticationValidator = new ExclusiveValidator<>(map, authSelected); + + var authChoice = new ChoicePaneComp(entries, authSelected); + var authQ = new DynamicOptionsBuilder(I18n.observable("jdbc.authentication")) + .addComp((ObservableValue) null, authChoice, authSelected) + .bindChoice( + () -> { + if (entries.indexOf(authSelected.get()) == 0) { + return passwordAuthProp; + } + if (entries.indexOf(authSelected.get()) == 1) { + return new SimpleObjectProperty(new WindowsAuth()); + } + return null; + }, + authProp) + .buildComp(); + + store.bind(Bindings.createObjectBinding( + () -> { + return MssqlSimpleStore.builder() + .address(addrProp.getValue()) + .auth(authProp.getValue()) + .build(); + }, + addrProp, + authProp)); + + return new GuiDialog( + new VerticalComp(List.of(addrQ, authQ)), + new ChainedValidator(List.of(addressValidator, authenticationValidator))); + } + + @Override + public String getDisplayIconFileName() { + return "jdbc:mssql_icon.svg"; + } + + @Override + public DataStore defaultStore() { + return MssqlSimpleStore.builder() + .address(MssqlAddress.builder() + .hostname("localhost") + .port(DEFAULT_PORT) + .build()) + .auth(new SimpleAuthMethod(DEFAULT_USERNAME, null)) + .build(); + } + + @Override + public List getPossibleNames() { + return List.of("mssql", "sqlserver", "microsoft sql", "microsoft sql server"); + } + + @Override + public List> getStoreClasses() { + return List.of(MssqlSimpleStore.class, MssqlUrlStore.class); + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlUrlStore.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlUrlStore.java new file mode 100644 index 00000000..e287ee78 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/mssql/MssqlUrlStore.java @@ -0,0 +1,34 @@ +package io.xpipe.ext.jdbcx.mssql; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.store.ShellStore; +import io.xpipe.ext.jdbc.JdbcUrlStore; +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +@JsonTypeName("mssqlUrl") +@SuperBuilder +@Jacksonized +@Getter +public class MssqlUrlStore extends JdbcUrlStore implements MssqlStore { + + @Builder.Default + protected ShellStore proxy = ShellStore.local(); + + public MssqlUrlStore(ShellStore proxy, String url) { + super(url); + this.proxy = proxy; + } + + @Override + public String getAddress() { + return getUrl().substring(0, getUrl().indexOf(";")); + } + + @Override + protected String getProtocol() { + return MssqlStoreProvider.PROTOCOL; + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleInstall.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleInstall.java new file mode 100644 index 00000000..f776a6a5 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleInstall.java @@ -0,0 +1,40 @@ +package io.xpipe.ext.jdbcx.oracle; + +import io.xpipe.extension.DownloadModuleInstall; +import io.xpipe.extension.util.HttpHelper; +import org.rauschig.jarchivelib.Archiver; +import org.rauschig.jarchivelib.ArchiverFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; + +public class OracleInstall extends DownloadModuleInstall { + + public OracleInstall() { + super( + "oracle", + "io.xpipe.ext.jdbc", + "oracle_license.txt", + "https://www.oracle.com/database/technologies/appdev/jdbc-downloads.html", + List.of("mysql-connector-j-8.0.31.jar")); + } + + @Override + public void installInternal(Path directory) throws Exception { + var file = HttpHelper.downloadFile( + "https://download.oracle.com/otn-pub/otn_software/jdbc/218/ojdbc11-full.tar.gz"); + Archiver archiver = ArchiverFactory.createArchiver("tar", "gz"); + var temp = Files.createTempDirectory(null); + archiver.extract(file.toFile(), temp.toFile()); + + var content = temp.resolve("ojdbc11-full"); + Files.delete(content.resolve("ojdbc11_g.jar")); + Files.delete(content.resolve("ojdbc11dms.jar")); + Files.delete(content.resolve("ojdbc11dms_g.jar")); + Files.delete(content.resolve("xmlparserv2_sans_jaxp_services.jar")); + + Files.move(content, directory, StandardCopyOption.REPLACE_EXISTING); + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleStandardStore.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleStandardStore.java new file mode 100644 index 00000000..1a57c46a --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleStandardStore.java @@ -0,0 +1,47 @@ +package io.xpipe.ext.jdbcx.oracle; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.store.ShellStore; +import io.xpipe.ext.jdbc.JdbcBaseStore; +import io.xpipe.ext.jdbc.JdbcDatabaseStore; +import io.xpipe.ext.jdbc.address.JdbcAddress; +import io.xpipe.ext.jdbc.auth.AuthMethod; +import io.xpipe.ext.jdbc.auth.SimpleAuthMethod; +import io.xpipe.ext.jdbc.auth.WindowsAuth; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +import java.util.HashMap; +import java.util.Map; + +@JsonTypeName("oracleStandard") +@SuperBuilder +@Jacksonized +public class OracleStandardStore extends JdbcDatabaseStore implements JdbcBaseStore { + + public OracleStandardStore(ShellStore proxy, JdbcAddress address, AuthMethod auth, String database) { + super(proxy, address, auth, database); + } + + @Override + public String toUrl() { + var base = "jdbc:oracle://" + address.toAddressString() + "/" + database; + return base; + } + + @Override + public Map createProperties() { + var p = new HashMap(); + + switch (auth) { + case SimpleAuthMethod s -> { + p.put("user", s.getUsername()); + p.put("password", s.getPassword().getSecretValue()); + } + case WindowsAuth a -> {} + default -> {} + } + + return p; + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleStoreProvider.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleStoreProvider.java new file mode 100644 index 00000000..dccfe8af --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleStoreProvider.java @@ -0,0 +1,220 @@ +package io.xpipe.ext.jdbcx.oracle; + +import io.xpipe.core.store.DataStore; +import io.xpipe.core.store.ShellStore; +import io.xpipe.ext.jdbc.JdbcGuiHelper; +import io.xpipe.ext.jdbc.JdbcStoreProvider; +import io.xpipe.ext.jdbc.address.JdbcBasicAddress; +import io.xpipe.ext.jdbc.auth.AuthMethod; +import io.xpipe.ext.jdbc.auth.SimpleAuthMethod; +import io.xpipe.ext.jdbc.auth.WindowsAuth; +import io.xpipe.ext.jdbc.postgres.PostgresUrlStore; +import io.xpipe.extension.GuiDialog; +import io.xpipe.extension.I18n; +import io.xpipe.extension.ModuleInstall; +import io.xpipe.extension.fxcomps.Comp; +import io.xpipe.extension.fxcomps.impl.ChoicePaneComp; +import io.xpipe.extension.fxcomps.impl.ShellStoreChoiceComp; +import io.xpipe.extension.fxcomps.impl.TabPaneComp; +import io.xpipe.extension.fxcomps.impl.VerticalComp; +import io.xpipe.extension.util.*; +import javafx.beans.binding.Bindings; +import javafx.beans.property.Property; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.value.ObservableValue; +import javafx.scene.layout.Region; + +import java.util.List; +import java.util.Map; + +public class OracleStoreProvider extends JdbcStoreProvider { + + public static final String PROTOCOL = "oracle:thin"; + public static final int DEFAULT_PORT = 1521; + public static final String DEFAULT_USERNAME = "oracle"; + + public OracleStoreProvider() { + super("oracle.jdbc.driver.OracleDriver"); + } + + @Override + public boolean init() throws Exception { + super.init(); + return false; + } + + @Override + public ModuleInstall getRequiredAdditionalInstallation() { + return new OracleInstall(); + } + + @Override + public GuiDialog guiDialog(Property store) { + var wizVal = new SimpleValidator(); + var wizValue = new SimpleObjectProperty( + store.getValue() instanceof OracleStandardStore ? store.getValue() : null); + var wizard = new TabPaneComp.Entry(I18n.observable("jdbc.connectionWizard"), null, wizard(wizValue, wizVal)); + + var urlVal = new SimpleValidator(); + var urlValue = + new SimpleObjectProperty<>(store.getValue() instanceof PostgresUrlStore ? store.getValue() : null); + var url = new TabPaneComp.Entry(I18n.observable("jdbc.connectionUrl"), null, url(urlValue, urlVal)); + + var stringVal = new SimpleValidator(); + var stringValue = new SimpleObjectProperty<>(store.getValue()); + var string = + new TabPaneComp.Entry(I18n.observable("jdbc.connectionString"), null, string(stringValue, stringVal)); + + var selected = new SimpleObjectProperty<>(store.getValue() instanceof PostgresUrlStore ? url : wizard); + + var map = Map.of( + wizard, wizVal, + url, urlVal, + string, stringVal); + var orVal = new ExclusiveValidator<>(map, selected); + + var propMap = Map.of( + wizard, wizValue, + url, urlValue, + string, stringValue); + PropertiesHelper.bindExclusive(selected, propMap, store); + + var pane = new TabPaneComp(selected, List.of(wizard, url)); + return new GuiDialog(pane, orVal); + } + + private Comp string(Property store, Validator val) { + return Comp.of(() -> new Region()); + } + + private Comp url(Property store, Validator val) { + return JdbcGuiHelper.url(PROTOCOL, PostgresUrlStore.class, store, val); + } + + private Comp wizard(Property store, Validator val) { + OracleStandardStore st = (OracleStandardStore) store.getValue(); + + var addrProp = new SimpleObjectProperty<>(st != null ? (JdbcBasicAddress) st.getAddress() : null); + var databaseProp = + new SimpleStringProperty(store.getValue() instanceof OracleStandardStore s ? s.getDatabase() : null); + var host = new SimpleStringProperty( + addrProp.getValue() != null ? addrProp.getValue().getHostname() : null); + var port = new SimpleObjectProperty<>( + addrProp.getValue() != null ? addrProp.getValue().getPort() : null); + var proxyProperty = new SimpleObjectProperty<>(st.getProxy()); + var connectionGui = new DynamicOptionsBuilder(I18n.observable("jdbc.connection")) + .addString(I18n.observable("jdbc.host"), host) + .nonNull(val) + .addInteger(I18n.observable("jdbc.port"), port) + .bind( + () -> { + return JdbcBasicAddress.builder() + .hostname(host.get()) + .port(port.get()) + .build(); + }, + addrProp) + .addString(I18n.observable("jdbc.database"), databaseProp) + .nonNull(val) + .addComp("proxy", ShellStoreChoiceComp.proxy(proxyProperty), proxyProperty) + .buildComp(); + + Property authProp = new SimpleObjectProperty<>(st.getAuth()); + Property passwordAuthProp = new SimpleObjectProperty<>( + authProp.getValue() instanceof SimpleAuthMethod ? (SimpleAuthMethod) authProp.getValue() : null); + var passwordAuthQ = Comp.of(() -> { + var user = new SimpleStringProperty( + passwordAuthProp.getValue() != null + ? passwordAuthProp.getValue().getUsername() + : DEFAULT_USERNAME); + var pass = new SimpleObjectProperty<>( + passwordAuthProp.getValue() != null + ? passwordAuthProp.getValue().getPassword() + : null); + return new DynamicOptionsBuilder(false) + .addString(I18n.observable("jdbc.username"), user) + .nonNull(val) + .addSecret(I18n.observable("jdbc.password"), pass) + .nonNull(val) + .bind( + () -> { + return new SimpleAuthMethod(user.get(), pass.get()); + }, + passwordAuthProp) + .build(); + }); + + Comp authChoice; + var passwordEntry = new ChoicePaneComp.Entry(I18n.observable("jdbc.passwordAuth"), passwordAuthQ); + var windowsEntry = new ChoicePaneComp.Entry(I18n.observable("jdbc.windowsAuth"), Comp.of(Region::new)); + var entries = List.of(passwordEntry, windowsEntry); + var authSelected = new SimpleObjectProperty( + authProp.getValue() == null || authProp.getValue() instanceof SimpleAuthMethod + ? passwordEntry + : windowsEntry); + var check = Validator.nonNull(val, I18n.observable("jdbc.authentication"), authSelected); + authChoice = new ChoicePaneComp(entries, authSelected).apply(s -> check.decorates(s.get())); + var authQ = new DynamicOptionsBuilder(I18n.observable("jdbc.authentication")) + .addComp((ObservableValue) null, authChoice, authSelected) + .bindChoice( + () -> { + if (entries.indexOf(authSelected.get()) == 0) { + return passwordAuthProp; + } + if (entries.indexOf(authSelected.get()) == 1) { + return new SimpleObjectProperty(new WindowsAuth()); + } + return null; + }, + authProp) + .buildComp(); + + store.bind(Bindings.createObjectBinding( + () -> { + return new OracleStandardStore( + proxyProperty.get(), addrProp.getValue(), authProp.getValue(), databaseProp.get()); + }, + proxyProperty, + addrProp, + databaseProp, + authProp)); + + return new VerticalComp(List.of(connectionGui, authQ)); + } + + @Override + public List> getStoreClasses() { + return List.of(OracleStandardStore.class, OracleUrlStore.class); + } + + @Override + public String getDisplayIconFileName() { + return "jdbc:oracle_icon.svg"; + } + + private OracleUrlStore defaultUrlStore() { + return OracleUrlStore.builder().build(); + } + + private OracleStandardStore defaultSimpleStore() { + return defaultStore().asNeeded(); + } + + @Override + public DataStore defaultStore() { + return new OracleStandardStore( + ShellStore.local(), + JdbcBasicAddress.builder() + .hostname("localhost") + .port(DEFAULT_PORT) + .build(), + new SimpleAuthMethod(DEFAULT_USERNAME, null), + DEFAULT_USERNAME); + } + + @Override + public List getPossibleNames() { + return List.of("oracle", "oraclesql", "osql"); + } +} diff --git a/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleUrlStore.java b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleUrlStore.java new file mode 100644 index 00000000..3b1459f9 --- /dev/null +++ b/ext/jdbcx/src/main/java/io/xpipe/ext/jdbcx/oracle/OracleUrlStore.java @@ -0,0 +1,35 @@ +package io.xpipe.ext.jdbcx.oracle; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.store.ShellStore; +import io.xpipe.ext.jdbc.JdbcBaseStore; +import io.xpipe.ext.jdbc.JdbcUrlStore; +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +@JsonTypeName("oracleUrl") +@SuperBuilder +@Jacksonized +@Getter +public class OracleUrlStore extends JdbcUrlStore implements JdbcBaseStore { + + @Builder.Default + protected ShellStore proxy = ShellStore.local(); + + public OracleUrlStore(ShellStore proxy, String url) { + super(url); + this.proxy = proxy; + } + + @Override + public String getAddress() { + return getUrl().substring(0, getUrl().indexOf("/")); + } + + @Override + protected String getProtocol() { + return OracleStoreProvider.PROTOCOL; + } +} diff --git a/ext/jdbcx/src/main/java/module-info.java b/ext/jdbcx/src/main/java/module-info.java index 50f70967..9d27d99d 100644 --- a/ext/jdbcx/src/main/java/module-info.java +++ b/ext/jdbcx/src/main/java/module-info.java @@ -1 +1,35 @@ -module io.xpipe.ext.jdbcx {} +import com.fasterxml.jackson.databind.Module; +import io.xpipe.ext.jdbc.JdbcDialect; +import io.xpipe.ext.jdbcx.JdbcxJacksonModule; +import io.xpipe.ext.jdbcx.mssql.MssqlDialect; +import io.xpipe.ext.jdbcx.mssql.MssqlStoreProvider; +import io.xpipe.ext.jdbcx.oracle.OracleStoreProvider; +import io.xpipe.extension.DataStoreProvider; + +import java.sql.Driver; + +open module io.xpipe.ext.jdbcx { + exports io.xpipe.ext.jdbcx.mssql; + + requires io.xpipe.ext.jdbc; + requires io.xpipe.core; + requires io.xpipe.extension; + requires static jarchivelib; + requires static lombok; + requires java.sql; + requires com.fasterxml.jackson.databind; + requires static net.synedra.validatorfx; + requires javafx.base; + requires javafx.graphics; + requires com.microsoft.sqlserver.jdbc; + requires io.xpipe.beacon; + + uses Driver; + + provides JdbcDialect with MssqlDialect; + provides Module with + JdbcxJacksonModule; + provides DataStoreProvider with + MssqlStoreProvider, + OracleStoreProvider; +} diff --git a/ext/office/build.gradle b/ext/office/build.gradle index fce335e8..de17c96c 100644 --- a/ext/office/build.gradle +++ b/ext/office/build.gradle @@ -1 +1,39 @@ -plugins { id 'java' } +plugins { + id 'java' + id "org.moditect.gradleplugin" version "1.0.0-rc3" +} + +dependencies { + implementation('org.apache.poi:poi-ooxml:5.2.3') { + exclude group: 'org.apache.commons', module: 'commons-collections4' + exclude group: 'org.apache.commons', module: 'commons-math3' + exclude group: 'commons-io', module: 'commons-io' + exclude group: 'org.apache.commons', module: 'commons-lang3' + } + implementation files("$buildDir/generated-modules/SparseBitSet-1.2.jar") + implementation files("$buildDir/generated-modules/commons-collections4-4.4.jar") +} + +apply from: "$rootDir/gradle/gradle_scripts/commons.gradle" +apply from: "$rootDir/gradle/gradle_scripts/extension.gradle" + +configurations { + compileOnly.extendsFrom(dep) + testImplementation.extendsFrom(dep) +} + +addDependenciesModuleInfo { + overwriteExistingFiles = true + jdepsExtraArgs = ['-q'] + outputDirectory = file("$buildDir/generated-modules") + modules { + module { + artifact 'com.zaxxer:SparseBitSet:1.2' + moduleInfoSource = ''' + module SparseBitSet { + exports com.zaxxer.sparsebits; + } + ''' + } + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxProvider.java b/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxProvider.java new file mode 100644 index 00000000..ae20e682 --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxProvider.java @@ -0,0 +1,71 @@ +package io.xpipe.ext.office.docx; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.dialog.Dialog; +import io.xpipe.core.source.*; +import io.xpipe.core.store.DataStore; +import io.xpipe.core.store.StreamDataStore; +import io.xpipe.ext.base.SimpleFileDataSourceProvider; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +import java.util.List; +import java.util.Map; + +public class DocxProvider implements SimpleFileDataSourceProvider { + + @Override + public Dialog configDialog(Source source, boolean all) { + return null; + } + + @Override + public DataSourceType getPrimaryType() { + return DataSourceType.TEXT; + } + + @Override + public Map> getSupportedExtensions() { + return Map.of(i18nKey("fileName"), List.of("docx")); + } + + @Override + public Source createDefaultSource(DataStore input) throws Exception { + return Source.builder().store(input.asNeeded()).build(); + } + + @Override + public Class getSourceClass() { + return Source.class; + } + + @Override + public List getPossibleNames() { + return List.of("docx"); + } + + @JsonTypeName("docx") + @SuperBuilder + @Jacksonized + public static class Source extends TextDataSource { + + @Override + protected TextWriteConnection newWriteConnection(WriteMode mode) { + var sup = super.newWriteConnection(mode); + if (sup != null) { + return sup; + } + + if (mode.equals(WriteMode.REPLACE)) { + return new DocxWriteConnection(); + } + + throw new UnsupportedOperationException(mode.getId()); + } + + @Override + protected TextReadConnection newReadConnection() { + return new DocxReadConnection(getStore()); + } + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxReadConnection.java b/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxReadConnection.java new file mode 100644 index 00000000..827ecb2a --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxReadConnection.java @@ -0,0 +1,43 @@ +package io.xpipe.ext.office.docx; + +import io.xpipe.core.source.DataSourceConnection; +import io.xpipe.core.source.TextReadConnection; +import io.xpipe.core.store.StreamDataStore; +import org.apache.poi.xwpf.extractor.XWPFWordExtractor; +import org.apache.poi.xwpf.usermodel.XWPFDocument; + +import java.util.Arrays; +import java.util.stream.Stream; + +public class DocxReadConnection implements TextReadConnection { + + private final StreamDataStore store; + + public DocxReadConnection(StreamDataStore store) { + this.store = store; + } + + @Override + public void init() throws Exception {} + + @Override + public Stream lines() throws Exception { + try (XWPFDocument doc = new XWPFDocument(store.openInput())) { + + XWPFWordExtractor xwpfWordExtractor = new XWPFWordExtractor(doc); + String docText = xwpfWordExtractor.getText(); + return Arrays.stream(docText.split("\r\n")); + } + } + + @Override + public boolean canRead() throws Exception { + return store.canOpen(); + } + + @Override + public void forward(DataSourceConnection con) throws Exception {} + + @Override + public void close() throws Exception {} +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxWriteConnection.java b/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxWriteConnection.java new file mode 100644 index 00000000..0caf4e5b --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/docx/DocxWriteConnection.java @@ -0,0 +1,14 @@ +package io.xpipe.ext.office.docx; + +import io.xpipe.core.source.TextWriteConnection; + +public class DocxWriteConnection implements TextWriteConnection { + @Override + public void init() throws Exception {} + + @Override + public void close() throws Exception {} + + @Override + public void writeLine(String line) throws Exception {} +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelDetector.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelDetector.java new file mode 100644 index 00000000..ce243559 --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelDetector.java @@ -0,0 +1,132 @@ +package io.xpipe.ext.office.excel; + +import io.xpipe.core.store.StreamDataStore; +import io.xpipe.ext.office.excel.model.ExcelCellLocation; +import io.xpipe.ext.office.excel.model.ExcelHeaderState; +import io.xpipe.ext.office.excel.model.ExcelRange; +import io.xpipe.ext.office.excel.model.ExcelSheetIdentifier; +import org.apache.poi.EmptyFileException; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.util.CellRangeAddress; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.StreamSupport; + +public class ExcelDetector { + + public static ExcelSource defaultSource(StreamDataStore store) { + return ExcelSource.builder() + .store(store) + .identifier(ExcelSheetIdentifier.builder().name("Sheet1").index(0).length(1).build()) + .headerState(ExcelHeaderState.INCLUDED) + .continueSelection(true) + .build(); + } + + public static ExcelSource detect(StreamDataStore store) throws Exception { + if (!store.canOpen()) { + return defaultSource(store); + } + + try (Workbook workbook = WorkbookFactory.create(store.openBufferedInput())) { + var sheets = ExcelHelper.getSheets(workbook); + var sheet = sheets.get(0); + var identifier = ExcelSheetIdentifier.builder().name(sheet.getSheetName()).index(0).length(sheets.size()).build(); + var state = ExcelHeaderState.INCLUDED; + var continueSelection = true; + var range = detectRange(sheet); + return ExcelSource.builder() + .store(store) + .continueSelection(continueSelection) + .identifier(identifier) + .headerState(state) + .range(range) + .build(); + } catch (EmptyFileException ex) { + return defaultSource(store); + } + } + + public static ExcelSource detect(StreamDataStore store, ExcelSheetIdentifier sheetId) throws Exception { + if (!store.canOpen()) { + return defaultSource(store); + } + + try (Workbook workbook = WorkbookFactory.create(store.openBufferedInput())) { + var sheets = ExcelHelper.getSheets(workbook); + var sheet = sheets.size() > 0 ? sheets.get(sheetId.getIndex()) : null; + var identifier = sheet != null ? ExcelSheetIdentifier.builder().name(sheet.getSheetName()).index(0).length(sheets.size()).build() : null; + var state = ExcelHeaderState.INCLUDED; + var continueSelection = true; + var range = sheet != null ? detectRange(sheet) : null; + return ExcelSource.builder() + .store(store) + .continueSelection(continueSelection) + .identifier(identifier) + .headerState(state) + .range(range) + .build(); + } + } + + private static ExcelRange detectRange(Sheet sheet) { + var rowsStart = 1; + var rowsEnd = sheet.getLastRowNum() + 1; + + var empty = StreamSupport.stream(sheet.spliterator(), false).findAny().isEmpty(); + if (empty) { + return null; + } + + for (Row cells : sheet) { + if (!isRowEmpty(cells) && !hasMergedRegions(sheet, cells)) { + break; + } + + rowsStart++; + } + + AtomicInteger columnStart = new AtomicInteger(Integer.MAX_VALUE); + AtomicInteger columnEnd = new AtomicInteger(1); + StreamSupport.stream(sheet.spliterator(), false).skip(rowsStart - 1).forEach(cells -> { + var s = getRowStart(cells); + if (s < columnStart.get()) { + columnStart.set(s); + } + + var e = (int) StreamSupport.stream(cells.spliterator(), false).count(); + if (e > columnEnd.get()) { + columnEnd.set(e); + } + }); + + return new ExcelRange( + new ExcelCellLocation(rowsStart, columnStart.get()), new ExcelCellLocation(rowsEnd, columnEnd.get())); + } + + private static boolean isRowEmpty(Row row) { + return StreamSupport.stream(row.spliterator(), false) + .allMatch(cell -> cell.getCellType() == CellType._NONE || cell.getCellType() == CellType.BLANK); + } + + private static boolean hasMergedRegions(Sheet sheet, Row row) { + int count = 0; + for (int i = 0; i < sheet.getNumMergedRegions(); ++i) { + CellRangeAddress range = sheet.getMergedRegion(i); + if (range.getFirstRow() <= row.getRowNum() && range.getLastRow() >= row.getRowNum()) ++count; + } + return count > 0; + } + + private static int getRowStart(Row row) { + var index = 1; + for (Cell cell : row) { + if (cell.getCellType() != CellType._NONE && cell.getCellType() != CellType.BLANK) { + break; + } + + index++; + } + return index; + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelHelper.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelHelper.java new file mode 100644 index 00000000..c669808a --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelHelper.java @@ -0,0 +1,78 @@ +package io.xpipe.ext.office.excel; + +import io.xpipe.core.store.StreamDataStore; +import io.xpipe.ext.office.excel.model.ExcelRange; +import io.xpipe.ext.office.excel.model.ExcelSheetIdentifier; +import org.apache.poi.EmptyFileException; +import org.apache.poi.ss.usermodel.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public class ExcelHelper { + + public static Stream> rowStream(Sheet sheet, ExcelRange range, boolean continuousSelection) { + return StreamSupport.stream(sheet.spliterator(), false) + .skip(range != null ? range.getBegin().getRow() - 1 : 0) + .limit( + range == null || continuousSelection + ? Integer.MAX_VALUE + : range.getEnd().getRow() - range.getBegin().getRow() + 1) + .map(cells -> StreamSupport.stream(cells.spliterator(), false) + .skip(range != null ? range.getBegin().getColumn() - 1 : 0) + .toList()) + .takeWhile(cells -> !cells.stream() + .allMatch( + cell -> cell.getCellType() == CellType._NONE || cell.getCellType() == CellType.BLANK)); + } + + public static List getSheets(Workbook workbook) { + var sheets = new ArrayList(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + sheets.add(workbook.getSheetAt(i)); + } + return sheets; + } + + public static ExcelSheetIdentifier getDefaultSelected( + ExcelSheetIdentifier identifier, List available) { + if (identifier == null) { + return available.size() > 0 ? available.get(0) : null; + } + + var byName = available.stream() + .filter(identifier1 -> identifier1.getName().equals(identifier.getName())) + .findFirst(); + if (byName.isPresent()) { + return byName.get(); + } + + return available.size() == identifier.getLength() ? available.get(identifier.getIndex()) : null; + } + + public static List getSheetIdentifiers(Workbook workbook) { + var sheets = getSheets(workbook); + return IntStream.range(0, sheets.size()) + .mapToObj(operand -> ExcelSheetIdentifier.builder() + .name(sheets.get(operand).getSheetName()) + .index(operand) + .length(sheets.size()) + .build()) + .toList(); + } + + public static List getSheetIdentifiers(StreamDataStore store) throws Exception { + if (!store.canOpen()) { + return List.of(); + } + + try (Workbook workbook = WorkbookFactory.create(store.openBufferedInput())) { + return getSheetIdentifiers(workbook); + } catch (EmptyFileException ex) { + return List.of(); + } + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelReadConnection.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelReadConnection.java new file mode 100644 index 00000000..a73e18ba --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelReadConnection.java @@ -0,0 +1,138 @@ +package io.xpipe.ext.office.excel; + +import io.xpipe.core.data.node.DataStructureNodeAcceptor; +import io.xpipe.core.data.node.TupleNode; +import io.xpipe.core.data.node.ValueNode; +import io.xpipe.core.data.type.TupleType; +import io.xpipe.core.data.type.ValueType; +import io.xpipe.core.impl.StreamReadConnection; +import io.xpipe.core.source.TableReadConnection; +import io.xpipe.ext.office.excel.model.ExcelHeaderState; +import io.xpipe.extension.util.DataTypeParser; +import org.apache.poi.EmptyFileException; +import org.apache.poi.ss.usermodel.*; + +import java.util.Collections; +import java.util.List; + +public class ExcelReadConnection extends StreamReadConnection implements TableReadConnection { + + private final ExcelSource source; + private Workbook workbook; + private Sheet sheet; + private TupleType type; + + public ExcelReadConnection(ExcelSource source) { + super(source.getStore(), null); + this.source = source; + } + + @Override + public void init() throws Exception { + super.init(); + try { + workbook = WorkbookFactory.create(inputStream); + } catch (EmptyFileException ex) { + return; + } + var sheets = ExcelHelper.getSheets(workbook); + sheet = sheets.stream() + .filter(s -> s.getSheetName().equals(source.getIdentifier().getName())) + .findFirst() + .orElse(workbook.getSheetAt(source.getIdentifier().getIndex())); + + if (source.getHeaderState() == ExcelHeaderState.INCLUDED) { + var names = ExcelHelper.rowStream(sheet, source.getRange(), false) + .findFirst() + .map(cells -> cells.stream() + .map(cell -> map(cell).asString().trim()) + .toList()) + .orElse(List.of()); + type = TupleType.of(names, Collections.nCopies(names.size(), ValueType.of())); + } else { + type = TupleType.of(Collections.nCopies( + source.getRange().getEnd().getColumn() + - source.getRange().getBegin().getColumn() + + 1, + ValueType.of())); + } + } + + @Override + public void close() throws Exception { + if (workbook != null) { + workbook.close(); + } + super.close(); + } + + @Override + public TupleType getDataType() { + return type; + } + + @Override + public void withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception { + if (workbook == null) { + return; + } + + var iterator = ExcelHelper.rowStream(sheet, source.getRange(), source.isContinueSelection()) + .skip(source.getHeaderState() == ExcelHeaderState.INCLUDED ? 1 : 0) + .iterator(); + while (iterator.hasNext()) { + var row = iterator.next(); + var t = row.stream().map(cell -> map(cell)).limit(type.getSize()).toList(); + var tuple = TupleNode.of(type.getNames(), t); + if (!lineAcceptor.accept(tuple)) { + break; + } + ; + } + } + + private ValueNode map(Cell cell) { + DataFormatter dataFormatter = new DataFormatter(); + dataFormatter.setUse4DigitYearsInAllDateFormats(true); + String rawValue = dataFormatter.formatCellValue(cell); + return switch (cell.getCellType()) { + case _NONE -> ValueNode.nullValue(); + case NUMERIC -> { + if (DateUtil.isCellDateFormatted(cell)) { + var date = cell.getDateCellValue(); + var instant = date.toInstant(); + yield ValueNode.ofDate(rawValue, instant); + } + + var monetary = DataTypeParser.parseMonetary(rawValue); + if (monetary.isPresent()) { + yield monetary.get(); + } + + var number = DataTypeParser.parseNumber(rawValue); + if (number.isPresent()) { + yield number.get(); + } + + yield ValueNode.ofDecimal(rawValue, cell.getNumericCellValue()); + } + case STRING -> { + yield ValueNode.ofText(cell.getStringCellValue()); + } + case FORMULA -> { + FormulaEvaluator evaluator = workbook.getCreationHelper().createFormulaEvaluator(); + evaluator.evaluateInCell(cell); + yield map(cell); + } + case BLANK -> { + yield ValueNode.nullValue(); + } + case BOOLEAN -> { + yield ValueNode.ofBoolean(cell.getBooleanCellValue()); + } + case ERROR -> { + yield ValueNode.nullValue(); + } + }; + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSource.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSource.java new file mode 100644 index 00000000..8e900e0f --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSource.java @@ -0,0 +1,53 @@ +package io.xpipe.ext.office.excel; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.source.TableDataSource; +import io.xpipe.core.source.TableReadConnection; +import io.xpipe.core.source.TableWriteConnection; +import io.xpipe.core.source.WriteMode; +import io.xpipe.core.store.StreamDataStore; +import io.xpipe.ext.office.excel.model.ExcelHeaderState; +import io.xpipe.ext.office.excel.model.ExcelRange; +import io.xpipe.ext.office.excel.model.ExcelSheetIdentifier; +import io.xpipe.extension.util.Validators; +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +@JsonTypeName("excel") +@SuperBuilder +@Jacksonized +@Getter +public class ExcelSource extends TableDataSource { + + private final ExcelSheetIdentifier identifier; + private final ExcelRange range; + private final ExcelHeaderState headerState; + private final boolean continueSelection; + + @Override + public void checkComplete() throws Exception { + super.checkComplete(); + Validators.nonNull(identifier, "Sheet"); + Validators.nonNull(headerState, "Header"); + } + + @Override + protected TableReadConnection newReadConnection() { + return new ExcelReadConnection(this); + } + + @Override + public TableWriteConnection newWriteConnection(WriteMode mode) { + var sup = super.newWriteConnection(mode); + if (sup != null) { + return sup; + } + + if (mode.equals(WriteMode.REPLACE)) { + return new ExcelWriteConnection(this); + } + + throw new UnsupportedOperationException(mode.getId()); + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSourceOpenAction.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSourceOpenAction.java new file mode 100644 index 00000000..f088bd12 --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSourceOpenAction.java @@ -0,0 +1,56 @@ +package io.xpipe.ext.office.excel; + +import io.xpipe.core.impl.FileStore; +import io.xpipe.core.process.OsType; +import io.xpipe.extension.DataSourceActionProvider; +import io.xpipe.extension.I18n; +import io.xpipe.extension.util.WindowsRegistry; +import javafx.beans.value.ObservableValue; + +import java.nio.file.Path; + +public class ExcelSourceOpenAction implements DataSourceActionProvider { + + @Override + public boolean isActive() throws Exception { + if (!(OsType.getLocal() == OsType.WINDOWS)) { + return false; + } + + return true; + } + + @Override + public boolean isApplicable(ExcelSource o) throws Exception { + return o.getStore() instanceof FileStore store && store.isLocal(); + } + + @Override + public void execute(ExcelSource store) throws Exception { + var locationString = WindowsRegistry.readString( + WindowsRegistry.HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\App Paths\\excel.exe", + null); + if (locationString.isEmpty()) { + return; + } + + var excelExecutable = Path.of(locationString.get()); + Runtime.getRuntime().exec(new String[] {excelExecutable.toString(), ((FileStore) store.getStore()).getFile()}); + } + + @Override + public Class getApplicableClass() { + return ExcelSource.class; + } + + @Override + public ObservableValue getName(ExcelSource store) { + return I18n.observable("openInExcel"); + } + + @Override + public String getIcon(ExcelSource store) { + return "mdi2m-microsoft-excel"; + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSourceProvider.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSourceProvider.java new file mode 100644 index 00000000..c699831c --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelSourceProvider.java @@ -0,0 +1,166 @@ +package io.xpipe.ext.office.excel; + +import io.xpipe.core.dialog.Dialog; +import io.xpipe.core.dialog.QueryConverter; +import io.xpipe.core.source.DataSourceType; +import io.xpipe.core.store.DataStore; +import io.xpipe.core.store.StreamDataStore; +import io.xpipe.ext.base.SimpleFileDataSourceProvider; +import io.xpipe.ext.office.excel.model.ExcelHeaderState; +import io.xpipe.ext.office.excel.model.ExcelRange; +import io.xpipe.ext.office.excel.model.ExcelSheetIdentifier; +import io.xpipe.extension.I18n; +import io.xpipe.extension.util.DialogHelper; +import io.xpipe.extension.util.DynamicOptionsBuilder; +import javafx.beans.property.Property; +import javafx.beans.property.SimpleBooleanProperty; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.value.ObservableValue; +import javafx.scene.layout.Region; +import org.apache.poi.openxml4j.util.ZipSecureFile; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +public class ExcelSourceProvider implements SimpleFileDataSourceProvider { + + @Override + public void init() throws Exception { + SimpleFileDataSourceProvider.super.init(); + + ZipSecureFile.setMinInflateRatio(0.001); + } + + @Override + public DataSourceType getPrimaryType() { + return DataSourceType.TABLE; + } + + @Override + public Map> getSupportedExtensions() { + return Map.of(i18nKey("fileName"), List.of("xlsx")); + } + + @Override + public Region configGui(Property source, boolean preferQuiet) throws Exception { + var s = source.getValue(); + + var headerState = new SimpleObjectProperty(s.getHeaderState()); + var headerStateNames = new LinkedHashMap>(); + headerStateNames.put(ExcelHeaderState.INCLUDED, I18n.observable("excel.included")); + headerStateNames.put(ExcelHeaderState.EXCLUDED, I18n.observable("excel.excluded")); + + var range = + new SimpleObjectProperty(source.getValue().getRange().toString()); + + var availableSheets = ExcelHelper.getSheetIdentifiers(source.getValue().getStore()); + var sheetNames = new LinkedHashMap>(); + availableSheets.forEach(identifier -> { + sheetNames.put( + identifier, + new SimpleStringProperty(identifier.getName() + " (" + (identifier.getIndex() + 1) + ".)")); + }); + var sheet = new SimpleObjectProperty<>(source.getValue().getIdentifier()); + + var continueAfterSelection = new SimpleBooleanProperty(source.getValue().isContinueSelection()); + + return new DynamicOptionsBuilder() + .addChoice(sheet, I18n.observable("excel.sheet"), sheetNames, false) + .addString("excel.range", range, true) + .addToggle("excel.continueAfterSelection", continueAfterSelection) + .addToggle(headerState, I18n.observable("excel.header"), headerStateNames) + .bind( + () -> { + return ExcelSource.builder() + .store(source.getValue().getStore()) + .identifier(sheet.get()) + .headerState(headerState.get()) + .range(ExcelRange.parse(range.get())) + .continueSelection(continueAfterSelection.get()) + .build(); + }, + source) + .build(); + } + + @Override + public List getPossibleNames() { + return List.of("excel", "xlsx", ".xlsx"); + } + + public Dialog configDialog(ExcelSource source, boolean preferQuiet) { + AtomicReference editedSource = new AtomicReference<>(source); + var sheetQ = Dialog.lazy(() -> { + var availableSheets = new ArrayList<>(ExcelHelper.getSheetIdentifiers(source.getStore())); + if (availableSheets.size() == 0) { + availableSheets.add(source.getIdentifier()); + } + + return Dialog.skipIf( + Dialog.choice( + "Sheet", + o -> o.getName(), + true, + false, + source.getIdentifier(), + availableSheets.toArray(ExcelSheetIdentifier[]::new)), + () -> availableSheets.size() <= 1) + .onCompletion((ExcelSheetIdentifier id) -> { + if (id != editedSource.get().getIdentifier()) { + editedSource.set(ExcelDetector.detect(source.getStore(), id)); + } + }); + }); + + var rangeQ = Dialog.lazy(() -> DialogHelper.query( + "Range", + editedSource.get().getRange(), + false, + new QueryConverter<>() { + @Override + protected ExcelRange fromString(String s) { + return ExcelRange.parse(s); + } + + @Override + protected String toString(ExcelRange value) { + return value.toString(); + } + }, + preferQuiet)); + + var headerQ = Dialog.lazy(() -> Dialog.choice( + "Header", + (ExcelHeaderState h) -> h == ExcelHeaderState.INCLUDED ? "Included" : "Excluded", + true, + preferQuiet, + editedSource.get().getHeaderState(), + ExcelHeaderState.values())); + + var continueQ = Dialog.lazy(() -> DialogHelper.booleanChoice( + "Continue Selection", editedSource.get().isContinueSelection(), preferQuiet)); + + return Dialog.chain(Dialog.busy(), sheetQ, rangeQ, headerQ, continueQ).evaluateTo(() -> ExcelSource.builder() + .store(source.getStore()) + .range(rangeQ.getResult()) + .identifier(sheetQ.getResult()) + .continueSelection(continueQ.getResult()) + .headerState(headerQ.getResult()) + .build()); + } + + @Override + public ExcelSource createDefaultSource(DataStore input) throws Exception { + var stream = (StreamDataStore) input; + return ExcelDetector.detect(stream); + } + + @Override + public Class getSourceClass() { + return ExcelSource.class; + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelWriteConnection.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelWriteConnection.java new file mode 100644 index 00000000..dda7ff0f --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/ExcelWriteConnection.java @@ -0,0 +1,121 @@ +package io.xpipe.ext.office.excel; + +import io.xpipe.core.data.node.DataStructureNode; +import io.xpipe.core.data.node.DataStructureNodeAcceptor; +import io.xpipe.core.data.node.TupleNode; +import io.xpipe.core.data.node.ValueNode; +import io.xpipe.core.impl.SimpleTableWriteConnection; +import io.xpipe.core.impl.StreamWriteConnection; +import io.xpipe.core.source.TableMapping; +import io.xpipe.ext.office.excel.model.ExcelHeaderState; +import lombok.Getter; +import org.apache.poi.EmptyFileException; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import java.time.Instant; +import java.util.Date; +import java.util.List; + +public class ExcelWriteConnection extends StreamWriteConnection implements SimpleTableWriteConnection { + + @Getter + private final ExcelSource source; + + private Workbook workbook; + private Sheet sheet; + + private int counter; + private boolean headerWritten; + + public ExcelWriteConnection(ExcelSource source) { + super(source.getStore(), null); + this.source = source; + } + + @Override + public void init() throws Exception { + super.init(); + try { + workbook = source.getStore().canOpen() + ? WorkbookFactory.create(source.getStore().openBufferedInput()) + : new XSSFWorkbook(); + } catch (EmptyFileException ex) { + workbook = new XSSFWorkbook(); + } + + var sheets = ExcelHelper.getSheets(workbook); + if (sheets.size() == 0) { + sheets = List.of(workbook.createSheet(source.getIdentifier().getName())); + } + + sheet = sheets.stream() + .filter(s -> s.getSheetName().equals(source.getIdentifier().getName())) + .findFirst() + .orElse(workbook.getSheetAt(source.getIdentifier().getIndex())); + } + + @Override + public void close() throws Exception { + workbook.write(outputStream); + workbook.close(); + super.close(); + } + + private void writeHeader(TableMapping mapping) { + if (!headerWritten && source.getHeaderState() == ExcelHeaderState.INCLUDED) { + var row = sheet.createRow(counter++); + for (int i = 0; i < mapping.getOutputType().getSize(); i++) { + var offset = + source.getRange() != null ? source.getRange().getBegin().getColumn() - 1 + i : i; + var cell = row.createCell(offset); + cell.setCellValue(mapping.getOutputType().getNames().get(i)); + } + headerWritten = true; + } + } + + @Override + public DataStructureNodeAcceptor writeLinesAcceptor(TableMapping mapping) { + writeHeader(mapping); + return node -> { + var row = sheet.createRow(counter); + for (int i = 0; i < mapping.getOutputType().getSize(); i++) { + var offset = + source.getRange() != null ? source.getRange().getBegin().getColumn() - 1 + i : i; + var cell = row.createCell(offset); + writeValue(cell, node.at(mapping.inverseMap(i).orElseThrow()).asValue()); + } + counter++; + + return true; + }; + } + + private void writeValue(Cell cell, ValueNode node) { + if (node.hasMetaAttribute(DataStructureNode.IS_BOOLEAN)) { + cell.setCellValue(node.hasMetaAttribute(DataStructureNode.BOOLEAN_TRUE)); + } else if (node.hasMetaAttribute(DataStructureNode.IS_DATE)) { + cell.setCellValue(Date.from(Instant.parse(node.getMetaAttribute(DataStructureNode.DATE_VALUE)))); + + var styleDateFormat = workbook.createCellStyle(); + styleDateFormat.setDataFormat((short) 0xe); + cell.setCellStyle(styleDateFormat); + } else if (node.hasMetaAttribute(DataStructureNode.IS_CURRENCY)) { + cell.setCellValue(Double.parseDouble(node.getMetaAttribute(DataStructureNode.DECIMAL_VALUE))); + + var styleCurrencyFormat = workbook.createCellStyle(); + styleCurrencyFormat.setDataFormat((short) 0x7); + cell.setCellStyle(styleCurrencyFormat); + } else if (node.hasMetaAttribute(DataStructureNode.IS_INTEGER)) { + cell.setCellValue(Double.parseDouble(node.getMetaAttribute(DataStructureNode.INTEGER_VALUE))); + } else if (node.hasMetaAttribute(DataStructureNode.IS_DECIMAL)) { + cell.setCellValue(Double.parseDouble(node.getMetaAttribute(DataStructureNode.DECIMAL_VALUE))); + } else { + cell.setCellValue(node.asString()); + } + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelCellLocation.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelCellLocation.java new file mode 100644 index 00000000..f98ac4ef --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelCellLocation.java @@ -0,0 +1,37 @@ +package io.xpipe.ext.office.excel.model; + +import lombok.Value; +import org.apache.poi.ss.util.CellReference; + +import java.util.regex.Pattern; + +@Value +public class ExcelCellLocation { + + private static final Pattern ID_PATTERN = Pattern.compile("([a-zA-Z]+)(\\d+)"); + int row; + int column; + + public static ExcelCellLocation parse(String id) { + var m = ID_PATTERN.matcher(id); + if (!m.matches()) { + throw new IllegalArgumentException("Invalid cell id: " + id); + } + + var column = toColumnIndex(m.group(1)); + var row = Integer.parseInt(m.group(2)); + return new ExcelCellLocation(row, column); + } + + private static String fromColumnIndex(int index) { + return CellReference.convertNumToColString(index - 1); + } + + private static int toColumnIndex(String id) { + return CellReference.convertColStringToIndex(id) + 1; + } + + public String toString() { + return fromColumnIndex(getColumn()) + getRow(); + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelHeaderState.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelHeaderState.java new file mode 100644 index 00000000..bfbff623 --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelHeaderState.java @@ -0,0 +1,86 @@ +package io.xpipe.ext.office.excel.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.xpipe.core.data.node.ArrayNode; + +import java.util.List; +import java.util.regex.Pattern; + +public enum ExcelHeaderState { + @JsonProperty("included") + INCLUDED, + @JsonProperty("excluded") + EXCLUDED; + + public static ExcelHeaderState determine(ArrayNode ar) { + if (ar.size() == 1) { + return INCLUDED; + } + + for (int i = 0; i < ar.at(0).size(); i++) { + if (!matchesPotentialHeader(ar, i)) { + return INCLUDED; + } + } + + return EXCLUDED; + } + + private static boolean matchesPotentialHeader(ArrayNode ar, int col) { + var t = getForColumnData(ar, col); + var headerType = getForColumnHeader(ar, col); + return t.equals(headerType); + } + + private static GeneralType getForColumnHeader(ArrayNode ar, int col) { + for (var type : GeneralType.TYPES) { + if (!type.matches(ar.at(0).at(col).asString())) { + continue; + } + + return type; + } + + throw new IllegalStateException(); + } + + private static GeneralType getForColumnData(ArrayNode ar, int col) { + out: + for (var type : GeneralType.TYPES) { + for (int i = 1; i < ar.size(); i++) { + if (!type.matches(ar.at(i).at(col).asString())) { + continue out; + } + } + + return type; + } + + throw new IllegalStateException(); + } + + private static interface GeneralType { + + static List TYPES = List.of(new NumberType(), new TextType()); + + boolean matches(String s); + } + + private static class NumberType implements GeneralType { + + private static final Pattern PATTERN = Pattern.compile("^-?\\d*(\\.\\d+)?$"); + + @Override + public boolean matches(String s) { + return PATTERN.matcher(s).matches(); + } + } + + private static class TextType implements GeneralType { + + @Override + public boolean matches(String s) { + return true; + } + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelJacksonModule.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelJacksonModule.java new file mode 100644 index 00000000..63afe606 --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelJacksonModule.java @@ -0,0 +1,78 @@ +package io.xpipe.ext.office.excel.model; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +import java.io.IOException; + +public class ExcelJacksonModule extends SimpleModule { + + @Override + public void setupModule(SetupContext context) { + addSerializer(ExcelCellLocation.class, new CellSerializer()); + addDeserializer(ExcelCellLocation.class, new CellDeserializer()); + + addSerializer(ExcelRange.class, new RangeSerializer()); + addDeserializer(ExcelRange.class, new RangeDeserializer()); + + context.addSerializers(_serializers); + context.addDeserializers(_deserializers); + } + + public static class CellSerializer extends StdSerializer { + + public CellSerializer() { + super(ExcelCellLocation.class); + } + + @Override + public void serialize(ExcelCellLocation value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeString(value.toString()); + } + } + + public static class CellDeserializer extends StdDeserializer { + + public CellDeserializer() { + super(ExcelCellLocation.class); + } + + @Override + public ExcelCellLocation deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + return ExcelCellLocation.parse(node.textValue()); + } + } + + public static class RangeSerializer extends StdSerializer { + + public RangeSerializer() { + super(ExcelRange.class); + } + + @Override + public void serialize(ExcelRange value, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeString(value.toString()); + } + } + + public static class RangeDeserializer extends StdDeserializer { + + public RangeDeserializer() { + super(ExcelRange.class); + } + + @Override + public ExcelRange deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + return ExcelRange.parse(node.asText()); + } + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelRange.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelRange.java new file mode 100644 index 00000000..94e9b1b7 --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelRange.java @@ -0,0 +1,24 @@ +package io.xpipe.ext.office.excel.model; + +import lombok.Value; + +@Value +public class ExcelRange { + + ExcelCellLocation begin; + ExcelCellLocation end; + + public static ExcelRange parse(String s) { + if (s.contains(":")) { + var b = ExcelCellLocation.parse(s.split(":")[0]); + var e = ExcelCellLocation.parse(s.split(":")[1]); + return new ExcelRange(b, e); + } + + throw new IllegalArgumentException("Invalid excel range: " + s); + } + + public String toString() { + return begin.toString() + ":" + end.toString(); + } +} diff --git a/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelSheetIdentifier.java b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelSheetIdentifier.java new file mode 100644 index 00000000..9b976ccd --- /dev/null +++ b/ext/office/src/main/java/io/xpipe/ext/office/excel/model/ExcelSheetIdentifier.java @@ -0,0 +1,15 @@ +package io.xpipe.ext.office.excel.model; + +import lombok.Value; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +@Value +@Jacksonized +@SuperBuilder +public final class ExcelSheetIdentifier { + + private final String name; + private final int index; + private final int length; +} diff --git a/ext/office/src/main/java/module-info.java b/ext/office/src/main/java/module-info.java index 08e74492..ab747419 100644 --- a/ext/office/src/main/java/module-info.java +++ b/ext/office/src/main/java/module-info.java @@ -1 +1,31 @@ -module io.xpipe.ext.office {} +import com.fasterxml.jackson.databind.Module; +import io.xpipe.ext.office.docx.DocxProvider; +import io.xpipe.ext.office.excel.ExcelSourceOpenAction; +import io.xpipe.ext.office.excel.ExcelSourceProvider; +import io.xpipe.ext.office.excel.model.ExcelJacksonModule; +import io.xpipe.extension.DataSourceActionProvider; +import io.xpipe.extension.DataSourceProvider; + +open module io.xpipe.ext.office { + requires static org.apache.commons.io; + requires io.xpipe.core; + requires io.xpipe.extension; + requires static lombok; + requires static org.apache.poi.ooxml; + requires com.fasterxml.jackson.databind; + requires static javafx.base; + requires static javafx.controls; + requires io.xpipe.ext.base; + + exports io.xpipe.ext.office.excel; + exports io.xpipe.ext.office.excel.model; + exports io.xpipe.ext.office.docx; + + provides Module with + ExcelJacksonModule; + provides DataSourceActionProvider with + ExcelSourceOpenAction; + provides DataSourceProvider with + DocxProvider, + ExcelSourceProvider; +} diff --git a/ext/office/src/main/resources/io/xpipe/ext/office/resources/extension.properties b/ext/office/src/main/resources/io/xpipe/ext/office/resources/extension.properties new file mode 100644 index 00000000..42f48b7f --- /dev/null +++ b/ext/office/src/main/resources/io/xpipe/ext/office/resources/extension.properties @@ -0,0 +1 @@ +name=Office Formats \ No newline at end of file diff --git a/ext/office/src/main/resources/io/xpipe/ext/office/resources/img/docx_icon.png b/ext/office/src/main/resources/io/xpipe/ext/office/resources/img/docx_icon.png new file mode 100644 index 00000000..01556bd0 Binary files /dev/null and b/ext/office/src/main/resources/io/xpipe/ext/office/resources/img/docx_icon.png differ diff --git a/ext/office/src/main/resources/io/xpipe/ext/office/resources/img/excel_icon.png b/ext/office/src/main/resources/io/xpipe/ext/office/resources/img/excel_icon.png new file mode 100644 index 00000000..e2a522fc Binary files /dev/null and b/ext/office/src/main/resources/io/xpipe/ext/office/resources/img/excel_icon.png differ diff --git a/ext/office/src/main/resources/io/xpipe/ext/office/resources/lang/translations_en.properties b/ext/office/src/main/resources/io/xpipe/ext/office/resources/lang/translations_en.properties new file mode 100644 index 00000000..a3f2de4f --- /dev/null +++ b/ext/office/src/main/resources/io/xpipe/ext/office/resources/lang/translations_en.properties @@ -0,0 +1,14 @@ +excel.displayName=Excel +excel.displayDescription=Microsoft Excel Format +excel.fileName=Excel File +excel.included=Included +excel.excluded=Excluded +excel.header=Header +excel.range=Range +excel.sheet=Sheet +excel.continueAfterSelection=Continue Selection +openInExcel=Open in Excel + +docx.displayName=Word Document +docx.displayDescription=Microsoft Word Format +docx.fileName=Word File \ No newline at end of file diff --git a/ext/office/src/test/java/module-info.java b/ext/office/src/test/java/module-info.java new file mode 100644 index 00000000..3756c095 --- /dev/null +++ b/ext/office/src/test/java/module-info.java @@ -0,0 +1,10 @@ +open module io.xpipe.ext.office.test { + exports tests; + + requires io.xpipe.ext.office; + requires org.junit.jupiter.api; + requires org.junit.jupiter.params; + requires io.xpipe.core; + requires io.xpipe.extension; + requires io.xpipe.api; +} diff --git a/ext/office/src/test/java/tests/ExcelTest.java b/ext/office/src/test/java/tests/ExcelTest.java new file mode 100644 index 00000000..2e6d994d --- /dev/null +++ b/ext/office/src/test/java/tests/ExcelTest.java @@ -0,0 +1,112 @@ +package tests; + +import io.xpipe.core.data.node.TupleNode; +import io.xpipe.core.data.node.ValueNode; +import io.xpipe.core.impl.FileStore; +import io.xpipe.core.impl.LocalStore; +import io.xpipe.ext.office.excel.ExcelSource; +import io.xpipe.ext.office.excel.model.ExcelCellLocation; +import io.xpipe.ext.office.excel.model.ExcelHeaderState; +import io.xpipe.ext.office.excel.model.ExcelRange; +import io.xpipe.ext.office.excel.model.ExcelSheetIdentifier; +import io.xpipe.extension.util.DaemonExtensionTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.util.Calendar; +import java.util.Currency; +import java.util.GregorianCalendar; + +public class ExcelTest extends DaemonExtensionTest { + + @Test + public void testEmpty() throws Exception { + var source = getSource("excel", "empty.xlsx").asTable(); + var lines = source.readAll(); + + Assertions.assertEquals(lines.size(), 0); + + ExcelSource detected = source.getInternalSource().asNeeded(); + Assertions.assertEquals(ExcelHeaderState.INCLUDED, detected.getHeaderState()); + Assertions.assertEquals(ExcelSheetIdentifier.builder().name("Sheet1").index(0).length(1).build(), detected.getIdentifier()); + Assertions.assertNull(detected.getRange()); + } + + @Test + public void testTwoSheetsEmpty() throws Exception { + var source = getSource("excel", "two-sheets-empty.xlsx").asTable(); + var lines = source.readAll(); + + Assertions.assertEquals(lines.size(), 0); + + ExcelSource detected = source.getInternalSource().asNeeded(); + Assertions.assertEquals(ExcelHeaderState.INCLUDED, detected.getHeaderState()); + Assertions.assertEquals(ExcelSheetIdentifier.builder().name("sheet 1").index(0).length(2).build(), detected.getIdentifier()); + Assertions.assertNull(detected.getRange()); + } + + @Test + public void testFinancialSample() throws Exception { + var source = getSource("excel", "Financial Sample.xlsx").asTable(); + var lines = source.readAll(); + + Assertions.assertEquals(700, lines.size()); + Assertions.assertEquals( + TupleNode.builder() + .add("Segment", ValueNode.ofText("Government")) + .add("Country", ValueNode.ofText("Canada")) + .add("Product", ValueNode.ofText("Carretera")) + .add("Discount Band", ValueNode.ofText("None")) + .add("Units Sold", ValueNode.ofCurrency("$ 1,618.50", "1618.5", Currency.getInstance("USD"))) + .add("Manufacturing Price", ValueNode.ofCurrency("$ 3.00", "3", Currency.getInstance("USD"))) + .add("Sale Price", ValueNode.ofCurrency("$ 20.00", "20", Currency.getInstance("USD"))) + .add("Gross Sales", ValueNode.ofCurrency("$ 32,370.00", "32370", Currency.getInstance("USD"))) + .add("Discounts", ValueNode.ofCurrency("$ - 0", "-0", Currency.getInstance("USD"))) + .add("Sales", ValueNode.ofCurrency("$ 32,370.00", "32370", Currency.getInstance("USD"))) + .add("COGS", ValueNode.ofCurrency("$ 16,185.00", "16185", Currency.getInstance("USD"))) + .add("Profit", ValueNode.ofCurrency("$ 16,185.00", "16185", Currency.getInstance("USD"))) + .add( + "Date", + ValueNode.ofDate( + "1/1/2014", + new GregorianCalendar(2014, Calendar.JANUARY, 1) + .getTime() + .toInstant())) + .add("Month Number", ValueNode.ofInteger("1", "1")) + .add("Month Name", ValueNode.ofText("January")) + .add("Year", ValueNode.ofText("2014")) + .build(), + lines.at(0)); + + ExcelSource detected = source.getInternalSource().asNeeded(); + Assertions.assertEquals(ExcelHeaderState.INCLUDED, detected.getHeaderState()); + Assertions.assertEquals(ExcelSheetIdentifier.builder().name("Sheet1").index(0).length(2).build(), detected.getIdentifier()); + Assertions.assertEquals( + new ExcelRange(ExcelCellLocation.parse("A1"), ExcelCellLocation.parse("P701")), detected.getRange()); + } + + @Test + public void testFinancialSampleRoundabout() throws Exception { + var source = getSource("excel", "Financial Sample.xlsx").asTable(); + + var targetFile = Files.createTempFile(null, ".xlsx").toString(); + var target = + getSource("excel", new FileStore(new LocalStore(), targetFile)).asTable(); + + source.forwardTo(target); + var lines = target.readAll(); + Assertions.assertEquals(700, lines.size()); + } + + @Test + public void testImages() throws Exception { + var source = getSource("excel", "images.xlsx").asTable(); + var lines = source.readAll(); + + Assertions.assertEquals(19, lines.size()); + + ExcelSource detected = source.getInternalSource().asNeeded(); + Assertions.assertEquals(ExcelHeaderState.INCLUDED, detected.getHeaderState()); + } +} diff --git a/ext/office/src/test/resources/Financial Sample.xlsx b/ext/office/src/test/resources/Financial Sample.xlsx new file mode 100644 index 00000000..e7f41b65 Binary files /dev/null and b/ext/office/src/test/resources/Financial Sample.xlsx differ diff --git a/ext/office/src/test/resources/empty.xlsx b/ext/office/src/test/resources/empty.xlsx new file mode 100644 index 00000000..1066ed38 Binary files /dev/null and b/ext/office/src/test/resources/empty.xlsx differ diff --git a/ext/office/src/test/resources/images.xlsx b/ext/office/src/test/resources/images.xlsx new file mode 100644 index 00000000..3039a7cd Binary files /dev/null and b/ext/office/src/test/resources/images.xlsx differ diff --git a/ext/office/src/test/resources/two-sheets-empty.xlsx b/ext/office/src/test/resources/two-sheets-empty.xlsx new file mode 100644 index 00000000..38da5504 Binary files /dev/null and b/ext/office/src/test/resources/two-sheets-empty.xlsx differ diff --git a/private_extensions.txt b/private_extensions.txt deleted file mode 100644 index 53cf60f9..00000000 --- a/private_extensions.txt +++ /dev/null @@ -1,2 +0,0 @@ -jdbcx -office