CodeMirror/xpipe
1
0
Fork 0
mirror of https://github.com/xpipe-io/xpipe.git synced 2024-09-30 00:56:56 +13:00
Code Issues Projects Releases Packages Wiki Activity

Change structure of data sources, rework comps

Browse source
This commit is contained in:
Christopher Schnick 2022-06-06 23:59:57 +02:00
parent 948dcf33ef
commit f4f9c1d978
40 changed files with 719 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api/src/main/java/io/xpipe/api/impl/DataSourceImpl.java
View file

@ -60,7 +60,7 @@ public abstract class DataSourceImpl implements DataSource {
});
var configInstance = startRes.getConfig();
configInstance.getCurrentValues().putAll(config);
configInstance.getConfigInstance().getCurrentValues().putAll(config);
var endReq = ReadExecuteExchange.Request.builder()
.target(id).dataStore(store).config(configInstance).build();
XPipeConnection.execute(con -> {

3
beacon/src/main/java/io/xpipe/beacon/exchange/ReadPreparationExchange.java
View file

@ -37,6 +37,8 @@ public class ReadPreparationExchange implements MessageExchange<ReadPreparationE
@NonNull
StreamDataStore store;
boolean configureAll;
}
@Jacksonized
@ -45,6 +47,7 @@ public class ReadPreparationExchange implements MessageExchange<ReadPreparationE
public static class Response implements ResponseMessage {
String determinedType;
@NonNull
DataSourceConfigInstance config;
}
}

1
beacon/src/main/java/io/xpipe/beacon/exchange/cli/DialogExchange.java
View file

@ -38,7 +38,6 @@ public class DialogExchange implements MessageExchange<DialogExchange.Request, D
@Builder
@Value
public static class Response implements ResponseMessage {
DataSourceConfigInstance instance;
String errorMsg;
}
}

2
charsetter/build.gradle
View file

@ -7,6 +7,8 @@ apply from: "$rootDir/deps/java.gradle"
apply from: "$rootDir/deps/commons.gradle"
apply from: "$rootDir/deps/junit.gradle"
apply from: "$rootDir/deps/lombok.gradle"
apply from: "$rootDir/deps/jackson.gradle"
configurations {
compileOnly.extendsFrom(dep)

58
charsetter/src/main/java/io/xpipe/charsetter/Charsetter.java
View file

@ -1,14 +1,19 @@
package io.xpipe.charsetter;
import lombok.Value;
import org.apache.commons.io.ByteOrderMark;
import org.apache.commons.io.input.BOMInputStream;
import org.apache.commons.lang3.function.FailableBiConsumer;
import org.apache.commons.lang3.function.FailableConsumer;
import org.apache.commons.lang3.function.FailableSupplier;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.*;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
public class Charsetter {
@ -25,7 +30,13 @@ public class Charsetter {
}
}
public static Charset read(FailableSupplier<InputStream, Exception> in, FailableBiConsumer<InputStream, Charset, Exception> con) throws Exception {
@Value
public static class Result {
Charset charset;
NewLine newLine;
}
public static Result read(FailableSupplier<InputStream, Exception> in, FailableConsumer<InputStreamReader, Exception> con) throws Exception {
checkInit();
try (var is = in.get();
@ -34,20 +45,53 @@ public class Charsetter {
String charsetName = bom == null ? null : bom.getCharsetName();
var charset = charsetName != null ? Charset.forName(charsetName) : null;
bin.mark(MAX_BYTES);
var bytes = bin.readNBytes(MAX_BYTES);
bin.reset();
if (charset == null) {
bin.mark(MAX_BYTES);
var bytes = bin.readNBytes(MAX_BYTES);
bin.reset();
charset = inferCharset(bytes);
}
var nl = inferNewLine(bytes);
if (con != null) {
con.accept(bin, charset);
con.accept(new InputStreamReader(bin, charset));
}
return charset;
return new Result(charset, nl);
}
}
public static NewLine inferNewLine(byte[] content) {
Map<NewLine, Integer> count = new HashMap<>();
for (var nl : NewLine.values()) {
var nlBytes = nl.getNewLine().getBytes(StandardCharsets.UTF_8);
count.put(nl, count(content, nlBytes));
}
if (count.values().stream().allMatch(v -> v == 0)) {
return null;
}
return count.entrySet().stream().min(Comparator.comparingInt(Map.Entry::getValue))
.orElseThrow().getKey();
}
public static int count(byte[] outerArray, byte[] smallerArray) {
int count = 0;
for(int i = 0; i < outerArray.length - smallerArray.length+1; ++i) {
boolean found = true;
for(int j = 0; j < smallerArray.length; ++j) {
if (outerArray[i+j] != smallerArray[j]) {
found = false;
break;
}
}
if (found) {
count++;
}
}
return count;
}
public static Charset inferCharset(byte[] content) {
checkInit();

5
charsetter/src/main/java/io/xpipe/charsetter/CharsetterContext.java
View file

@ -3,6 +3,7 @@ package io.xpipe.charsetter;
import lombok.AllArgsConstructor;
import lombok.Value;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Locale;
@ -10,6 +11,10 @@ import java.util.Locale;
@AllArgsConstructor
public class CharsetterContext {
public static CharsetterContext empty() {
return new CharsetterContext(Charset.defaultCharset().name(), Locale.getDefault(), Locale.getDefault(), List.of());
}
String systemCharsetName;
Locale systemLocale;

41
charsetter/src/main/java/io/xpipe/charsetter/NewLine.java Normal file
View file

@ -0,0 +1,41 @@
package io.xpipe.charsetter;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Arrays;
public enum NewLine {
@JsonProperty("lf")
LF("\n", "lf"),
@JsonProperty("crlf")
CRLF("\r\n", "crlf");
public static NewLine platform() {
return Arrays.stream(values())
.filter(n -> n.getNewLine().equals(System.getProperty("line.separator")))
.findFirst().orElseThrow();
}
public static NewLine id(String id) {
return Arrays.stream(values())
.filter(n -> n.getId().equalsIgnoreCase(id))
.findFirst().orElseThrow();
}
private final String newLine;
private final String id;
NewLine(String newLine, String id) {
this.newLine = newLine;
this.id = id;
}
public String getNewLine() {
return newLine;
}
public String getId() {
return id;
}
}

1
charsetter/src/main/java/module-info.java
View file

@ -4,4 +4,5 @@ module io.xpipe.charsetter {
requires org.apache.commons.io;
requires org.apache.commons.lang3;
requires static lombok;
requires com.fasterxml.jackson.databind;
}

86
core/src/main/java/io/xpipe/core/config/ConfigConverter.java Normal file
View file

@ -0,0 +1,86 @@
package io.xpipe.core.config;
import java.nio.charset.Charset;
public abstract class ConfigConverter<T> {
public static final ConfigConverter<Charset> CHARSET = new ConfigConverter<Charset>() {
@Override
protected Charset fromString(String s) {
return Charset.forName(s);
}
@Override
protected String toString(Charset value) {
return value.name();
}
};
public static final ConfigConverter<String> STRING = new ConfigConverter<String>() {
@Override
protected String fromString(String s) {
return s;
}
@Override
protected String toString(String value) {
return value;
}
};
public static final ConfigConverter<Character> CHARACTER = new ConfigConverter<Character>() {
@Override
protected Character fromString(String s) {
if (s.length() != 1) {
throw new IllegalArgumentException("Invalid character: " + s);
}
return s.toCharArray()[0];
}
@Override
protected String toString(Character value) {
return value.toString();
}
};
public static final ConfigConverter<Boolean> BOOLEAN = new ConfigConverter<Boolean>() {
@Override
protected Boolean fromString(String s) {
if (s.equalsIgnoreCase("y") || s.equalsIgnoreCase("yes") || s.equalsIgnoreCase("true")) {
return true;
}
if (s.equalsIgnoreCase("n") || s.equalsIgnoreCase("no") || s.equalsIgnoreCase("false")) {
return false;
}
throw new IllegalArgumentException("Invalid boolean: " + s);
}
@Override
protected String toString(Boolean value) {
return value ? "yes" : "no";
}
};
public T convertFromString(String s) {
if (s == null) {
return null;
}
return fromString(s);
}
public String convertToString(T value) {
if (value == null) {
return null;
}
return toString(value);
}
protected abstract T fromString(String s);
protected abstract String toString(T value);
}

14
core/src/main/java/io/xpipe/core/config/ConfigOption.java
View file

@ -1,14 +0,0 @@
package io.xpipe.core.config;
import com.fasterxml.jackson.annotation.JsonCreator;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Value;
@Value
@Builder
@AllArgsConstructor(onConstructor_={@JsonCreator})
public class ConfigOption {
String name;
String key;
}

23
core/src/main/java/io/xpipe/core/config/ConfigOptionSet.java
View file

@ -1,23 +0,0 @@
package io.xpipe.core.config;
import com.fasterxml.jackson.annotation.JsonCreator;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Singular;
import lombok.Value;
import java.util.List;
@Value
@Builder
@AllArgsConstructor(onConstructor_={@JsonCreator})
public class ConfigOptionSet {
public static ConfigOptionSet empty() {
return new ConfigOptionSet(List.of());
}
@Singular
List<ConfigOption> options;
}

26
core/src/main/java/io/xpipe/core/config/ConfigOptionSetInstance.java
View file

@ -1,26 +0,0 @@
package io.xpipe.core.config;
import com.fasterxml.jackson.annotation.JsonCreator;
import lombok.AllArgsConstructor;
import lombok.Value;
import java.util.Map;
@Value
@AllArgsConstructor(onConstructor_={@JsonCreator})
public class ConfigOptionSetInstance {
/**
* The available configuration options.
*/
ConfigOptionSet configOptions;
/**
* The current configuration options that are set.
*/
Map<String, String> currentValues;
public boolean isComplete() {
return currentValues.size() == configOptions.getOptions().size();
}
}

26
core/src/main/java/io/xpipe/core/config/ConfigParameter.java Normal file
View file

@ -0,0 +1,26 @@
package io.xpipe.core.config;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor;
import lombok.Value;
@Value
@AllArgsConstructor
public class ConfigParameter {
String key;
@JsonCreator
public ConfigParameter(String key) {
this.key = key;
this.converter = null;
}
@JsonIgnore
ConfigConverter<?> converter;
@SuppressWarnings("unchecked")
public <T> ConfigConverter<T> getConverter() {
return (ConfigConverter<T>) converter;
}
}

54
core/src/main/java/io/xpipe/core/config/ConfigParameterSetInstance.java Normal file
View file

@ -0,0 +1,54 @@
package io.xpipe.core.config;
import com.fasterxml.jackson.annotation.JsonCreator;
import lombok.AllArgsConstructor;
import lombok.Value;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Value
@AllArgsConstructor(onConstructor_={@JsonCreator})
public class ConfigParameterSetInstance {
/**
* The available configuration parameters.
*/
List<ConfigParameter> configParameters;
/**
* The current configuration options that are set.
*/
Map<String, String> currentValues;
public ConfigParameterSetInstance(Map<ConfigParameter, Object> map) {
configParameters = map.keySet().stream().toList();
currentValues = map.entrySet().stream().collect(Collectors.toMap(
e -> e.getKey().getKey(),
e -> e.getKey().getConverter().convertToString(e.getValue())));
}
public <X, T extends Function<X,?>> ConfigParameterSetInstance(Map<ConfigParameter, T> map, Object v) {
configParameters = map.keySet().stream().toList();
currentValues = map.entrySet().stream().collect(Collectors.toMap(
e -> e.getKey().getKey(),
e -> e.getKey().getConverter().convertToString(apply(e.getValue(), v))));
}
@SuppressWarnings("unchecked")
private static <X, T extends Function<X,?>, V> Object apply(T func, Object v) {
return func.apply((X) v);
}
public void update(ConfigParameter p, String val) {
currentValues.put(p.getKey(), val);
}
public Map<ConfigParameter, Object> evaluate() {
return configParameters.stream().collect(Collectors.toMap(
p -> p,
p -> p.getConverter().convertFromString(currentValues.get(p.getKey()))));
}
}

4
core/src/main/java/io/xpipe/core/connection/Connection.java Normal file
View file

@ -0,0 +1,4 @@
package io.xpipe.core.connection;
public class Connection {
}

4
core/src/main/java/io/xpipe/core/data/node/ArrayNode.java
View file

@ -8,6 +8,10 @@ import java.util.stream.Collectors;
public abstract class ArrayNode extends DataStructureNode {
public static ArrayNode empty() {
return of(List.of());
}
public static ArrayNode of(DataStructureNode... dsn) {
return of(List.of(dsn));
}

18
core/src/main/java/io/xpipe/core/source/DataSource.java
View file

@ -4,7 +4,8 @@ import com.fasterxml.jackson.databind.util.TokenBuffer;
import io.xpipe.core.store.DataStore;
import io.xpipe.core.util.JacksonHelper;
import lombok.AllArgsConstructor;
import lombok.NonNull;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import java.util.Optional;
@ -15,19 +16,20 @@ import java.util.Optional;
*
* This instance is only valid in combination with its associated data store instance.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public abstract class DataSource<DS extends DataStore> {
@NonNull
protected DS store;
@SneakyThrows
@SuppressWarnings("unchecked")
public DataSource<DS> copy() {
public <T extends DataSource<DS>> T copy() {
var mapper = JacksonHelper.newMapper();
TokenBuffer tb = new TokenBuffer(mapper, false);
mapper.writeValue(tb, this);
return mapper.readValue(tb.asParser(), getClass());
return (T) mapper.readValue(tb.asParser(), getClass());
}
public DataSource<DS> withStore(DS store) {
@ -36,6 +38,10 @@ public abstract class DataSource<DS extends DataStore> {
return c;
}
public boolean isComplete() {
return true;
}
/**
* Casts this instance to the required type without checking whether a cast is possible.
*/
@ -63,6 +69,10 @@ public abstract class DataSource<DS extends DataStore> {
public abstract DataSourceConnection openWriteConnection() throws Exception;
public DataSourceConnection openAppendingWriteConnection() throws Exception {
throw new UnsupportedOperationException("Appending write is not supported");
}
public DS getStore() {
return store;
}

30
core/src/main/java/io/xpipe/core/source/DataSourceConfigInstance.java
View file

@ -1,12 +1,13 @@
package io.xpipe.core.source;
import com.fasterxml.jackson.annotation.JsonCreator;
import io.xpipe.core.config.ConfigOptionSet;
import io.xpipe.core.config.ConfigOptionSetInstance;
import io.xpipe.core.config.ConfigParameter;
import io.xpipe.core.config.ConfigParameterSetInstance;
import lombok.AllArgsConstructor;
import lombok.Value;
import java.util.Map;
import java.util.function.Function;
/**
* Represents the current configuration of a data source.
@ -17,7 +18,7 @@ import java.util.Map;
public class DataSourceConfigInstance {
public static DataSourceConfigInstance xpbt() {
return new DataSourceConfigInstance("xpbt", ConfigOptionSet.empty(), Map.of());
return new DataSourceConfigInstance("xpbt", new ConfigParameterSetInstance(Map.of()));
}
/**
@ -26,22 +27,21 @@ public class DataSourceConfigInstance {
String provider;
/**
* The available configuration options.
* The available configuration parameters.
*/
ConfigOptionSet configOptions;
ConfigParameterSetInstance configInstance;
/**
* The current configuration options that are set.
*/
Map<String, String> currentValues;
public boolean isComplete() {
return currentValues.size() == configOptions.getOptions().size();
public DataSourceConfigInstance(String provider, Map<ConfigParameter, Object> map) {
this.provider = provider;
this.configInstance = new ConfigParameterSetInstance(map);
}
public DataSourceConfigInstance(String provider, ConfigOptionSetInstance cInstance) {
public <X, T extends Function<X,?>> DataSourceConfigInstance(String provider, Map<ConfigParameter, T> map, Object val) {
this.provider = provider;
this.configOptions = cInstance.getConfigOptions();
this.currentValues = cInstance.getCurrentValues();
this.configInstance = new ConfigParameterSetInstance(map, val);
}
public Map<ConfigParameter, Object> evaluate() {
return configInstance.evaluate();
}
}

14
core/src/main/java/io/xpipe/core/source/TableDataSource.java
View file

@ -1,7 +1,13 @@
package io.xpipe.core.source;
import io.xpipe.core.store.DataStore;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
@Data
@EqualsAndHashCode(callSuper = true)
@NoArgsConstructor
public abstract class TableDataSource<DS extends DataStore> extends DataSource<DS> {
public TableDataSource(DS store) {
@ -29,7 +35,15 @@ public abstract class TableDataSource<DS extends DataStore> extends DataSource<D
return con;
}
public final TableWriteConnection openAppendingWriteConnection() throws Exception {
var con = newAppendingWriteConnection();
con.init();
return con;
}
protected abstract TableWriteConnection newWriteConnection();
protected abstract TableWriteConnection newAppendingWriteConnection();
protected abstract TableReadConnection newReadConnection();
}

30
core/src/main/java/io/xpipe/core/source/TableReadConnection.java
View file

@ -15,6 +15,30 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public interface TableReadConnection extends DataSourceReadConnection {
public static TableReadConnection empty() {
return new TableReadConnection() {
@Override
public TupleType getDataType() throws Exception {
return TupleType.empty();
}
@Override
public int getRowCount() throws Exception {
return 0;
}
@Override
public void withRows(DataStructureNodeAcceptor<TupleNode> lineAcceptor) throws Exception {
}
@Override
public ArrayNode readRows(int maxLines) throws Exception {
return ArrayNode.of();
}
};
}
/**
* Returns the data type of the table data.
*/
@ -53,9 +77,7 @@ public interface TableReadConnection extends DataSourceReadConnection {
}
default void forward(DataSourceConnection con) throws Exception {
try (var tCon = (TableWriteConnection) con) {
tCon.init();
withRows(tCon.writeLinesAcceptor());
}
var tCon = (TableWriteConnection) con;
withRows(tCon.writeLinesAcceptor());
}
}

2
core/src/main/java/io/xpipe/core/source/TextDataSource.java
View file

@ -1,9 +1,11 @@
package io.xpipe.core.source;
import io.xpipe.core.store.DataStore;
import lombok.NoArgsConstructor;
import java.util.concurrent.atomic.AtomicInteger;
@NoArgsConstructor
public abstract class TextDataSource<DS extends DataStore> extends DataSource<DS> {
private static final int MAX_LINE_READ = 1000;

6
core/src/main/java/io/xpipe/core/store/FileDataStore.java
View file

@ -10,5 +10,11 @@ public interface FileDataStore extends StreamDataStore {
var i = n.lastIndexOf('.');
return Optional.of(i != -1 ? n.substring(0, i) : n);
}
@Override
default boolean persistent() {
return true;
}
String getFileName();
}

79
core/src/main/java/io/xpipe/core/store/InputStreamDataStore.java
View file

@ -1,6 +1,7 @@
package io.xpipe.core.store;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -15,7 +16,6 @@ public class InputStreamDataStore implements StreamDataStore {
private final InputStream in;
private BufferedInputStream bufferedInputStream;
private boolean opened = false;
public InputStreamDataStore(InputStream in) {
this.in = in;
@ -23,13 +23,84 @@ public class InputStreamDataStore implements StreamDataStore {
@Override
public InputStream openInput() throws Exception {
if (opened) {
if (bufferedInputStream != null) {
bufferedInputStream.reset();
return bufferedInputStream;
}
opened = true;
bufferedInputStream = new BufferedInputStream(in);
return bufferedInputStream;
bufferedInputStream.mark(Integer.MAX_VALUE);
return new InputStream() {
@Override
public int read() throws IOException {
return bufferedInputStream.read();
}
@Override
public int read(byte[] b) throws IOException {
return bufferedInputStream.read(b);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
return bufferedInputStream.read(b, off, len);
}
@Override
public byte[] readAllBytes() throws IOException {
return bufferedInputStream.readAllBytes();
}
@Override
public byte[] readNBytes(int len) throws IOException {
return bufferedInputStream.readNBytes(len);
}
@Override
public int readNBytes(byte[] b, int off, int len) throws IOException {
return bufferedInputStream.readNBytes(b, off, len);
}
@Override
public long skip(long n) throws IOException {
return bufferedInputStream.skip(n);
}
@Override
public void skipNBytes(long n) throws IOException {
bufferedInputStream.skipNBytes(n);
}
@Override
public int available() throws IOException {
return bufferedInputStream.available();
}
@Override
public void close() throws IOException {
reset();
}
@Override
public synchronized void mark(int readlimit) {
bufferedInputStream.mark(readlimit);
}
@Override
public synchronized void reset() throws IOException {
bufferedInputStream.reset();
}
@Override
public boolean markSupported() {
return bufferedInputStream.markSupported();
}
@Override
public long transferTo(OutputStream out) throws IOException {
return bufferedInputStream.transferTo(out);
}
};
}
@Override

6
core/src/main/java/io/xpipe/core/store/LocalFileDataStore.java
View file

@ -10,6 +10,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.util.Optional;
@ -52,6 +53,11 @@ public class LocalFileDataStore implements FileDataStore {
return Files.newOutputStream(file);
}
@Override
public OutputStream openAppendingOutput() throws Exception {
return Files.newOutputStream(file, StandardOpenOption.APPEND);
}
@Override
public boolean exists() {
return Files.exists(file);

8
core/src/main/java/io/xpipe/core/store/StreamDataStore.java
View file

@ -45,5 +45,13 @@ public interface StreamDataStore extends DataStore {
throw new UnsupportedOperationException("Can't open store output");
}
default OutputStream openAppendingOutput() throws Exception {
throw new UnsupportedOperationException("Can't open store output");
}
boolean exists();
default boolean persistent() {
return false;
}
}

22
core/src/main/java/io/xpipe/core/store/URLDataStore.java Normal file
View file

@ -0,0 +1,22 @@
package io.xpipe.core.store;
import lombok.Value;
import java.io.InputStream;
import java.net.URL;
@Value
public class URLDataStore implements StreamDataStore {
URL url;
@Override
public InputStream openInput() throws Exception {
return url.openStream();
}
@Override
public boolean exists() {
return true;
}
}

1
core/src/main/java/module-info.java
View file

@ -18,6 +18,7 @@ module io.xpipe.core {
opens io.xpipe.core.data.typed;
exports io.xpipe.core.config;
opens io.xpipe.core.config;
exports io.xpipe.core.connection;
requires com.fasterxml.jackson.core;
requires com.fasterxml.jackson.databind;

2
extension/build.gradle
View file

@ -30,7 +30,9 @@ repositories {
}
dependencies {
compileOnly 'org.junit.jupiter:junit-jupiter-api:5.8.2'
implementation project(':core')
implementation project(':charsetter')
implementation 'io.xpipe:fxcomps:0.1'
implementation 'com.google.code.gson:gson:2.9.0'

118
extension/src/main/java/io/xpipe/extension/DataSourceProvider.java
View file

@ -1,14 +1,16 @@
package io.xpipe.extension;
import io.xpipe.core.config.ConfigOption;
import io.xpipe.core.config.ConfigOptionSet;
import io.xpipe.charsetter.NewLine;
import io.xpipe.core.config.ConfigConverter;
import io.xpipe.core.config.ConfigParameter;
import io.xpipe.core.source.DataSource;
import io.xpipe.core.source.DataSourceType;
import io.xpipe.core.store.DataStore;
import javafx.beans.property.Property;
import javafx.scene.layout.Region;
import lombok.SneakyThrows;
import lombok.Value;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@ -26,9 +28,19 @@ public interface DataSourceProvider<T extends DataSource<?>> {
return GeneralType.FILE;
}
if (getDatabaseProvider() != null) {
return GeneralType.DATABASE;
}
throw new ExtensionException("Provider has no general type");
}
@SneakyThrows
@SuppressWarnings("unchecked")
default T createDefault() {
return (T) getSourceClass().getDeclaredConstructors()[0].newInstance();
}
default boolean supportsConversion(T in, DataSourceType t) {
return false;
}
@ -37,7 +49,7 @@ public interface DataSourceProvider<T extends DataSource<?>> {
throw new ExtensionException();
}
default void init() {
default void init() throws Exception {
}
default String i18n(String key) {
@ -48,7 +60,7 @@ public interface DataSourceProvider<T extends DataSource<?>> {
return getId() + "." + key;
}
default Region createConfigOptions(DataStore input, Property<? extends DataSource<?>> source) {
default Region createConfigGui(Property<T> source) {
return null;
}
@ -75,27 +87,32 @@ public interface DataSourceProvider<T extends DataSource<?>> {
Map<String, List<String>> getFileExtensions();
}
interface DatabaseProvider {
}
interface ConfigProvider<T extends DataSource<?>> {
@Value
static class Parameter {
ConfigParameter parameter;
Object currentValue;
boolean required;
}
static <T extends DataSource<?>> ConfigProvider<T> empty(List<String> names, Function<DataStore, T> func) {
return new ConfigProvider<>() {
@Override
public ConfigOptionSet getConfig() {
return ConfigOptionSet.empty();
public void applyConfig(T source, Map<ConfigParameter, Object> values) {
}
@Override
public T toDescriptor(DataStore store, Map<String, String> values) {
return func.apply(store);
}
@Override
public Map<String, String> toConfigOptions(T source) {
public Map<ConfigParameter, Function<T, Object>> toCompleteConfig() {
return Map.of();
}
@Override
public Map<ConfigOption, Function<String, ?>> getConverters() {
public Map<ConfigParameter, Object> toRequiredReadConfig(T desc) {
return Map.of();
}
@ -106,52 +123,29 @@ public interface DataSourceProvider<T extends DataSource<?>> {
};
}
ConfigOption
CHARSET_OPTION = new ConfigOption("Charset", "charset");
Function<String, Charset>
CHARSET_CONVERTER = ConfigProvider.charsetConverter();
Function<Charset, String>
CHARSET_STRING = Charset::name;
ConfigParameter CHARSET = new ConfigParameter(
"charset", ConfigConverter.CHARSET);
static String booleanName(String name) {
return name + " (y/n)";
}
public static final ConfigConverter<NewLine> NEW_LINE_CONVERTER = new ConfigConverter<NewLine>() {
@Override
protected NewLine fromString(String s) {
return NewLine.id(s);
}
static Function<String, Boolean> booleanConverter() {
return s -> {
if (s.equalsIgnoreCase("y") || s.equalsIgnoreCase("yes")) {
return true;
}
@Override
protected String toString(NewLine value) {
return value.getId();
}
};
if (s.equalsIgnoreCase("n") || s.equalsIgnoreCase("no")) {
return false;
}
ConfigParameter NEWLINE = new ConfigParameter(
"newline", NEW_LINE_CONVERTER);
throw new IllegalArgumentException("Invalid boolean: " + s);
};
}
void applyConfig(T source, Map<ConfigParameter, Object> values);
static Function<String, Character> characterConverter() {
return s -> {
if (s.length() != 1) {
throw new IllegalArgumentException("Invalid character: " + s);
}
Map<ConfigParameter, Function<T, Object>> toCompleteConfig();
return s.toCharArray()[0];
};
}
static Function<String, Charset> charsetConverter() {
return Charset::forName;
}
ConfigOptionSet getConfig();
T toDescriptor(DataStore store, Map<String, String> values);
Map<String, String> toConfigOptions(T desc);
Map<ConfigOption, Function<String, ?>> getConverters();
Map<ConfigParameter, Object> toRequiredReadConfig(T desc);
List<String> getPossibleNames();
}
@ -190,6 +184,10 @@ public interface DataSourceProvider<T extends DataSource<?>> {
return null;
}
default DatabaseProvider getDatabaseProvider() {
return null;
}
default boolean hasDirectoryProvider() {
return false;
}
@ -206,16 +204,16 @@ public interface DataSourceProvider<T extends DataSource<?>> {
* Attempt to create a useful data source descriptor from a data store.
* The result does not need to be always right, it should only reflect the best effort.
*/
T createDefaultDescriptor(DataStore input) throws Exception;
T createDefaultSource(DataStore input) throws Exception;
default T createDefaultWriteDescriptor(DataStore input) throws Exception {
return createDefaultDescriptor(input);
default T createDefaultWriteSource(DataStore input) throws Exception {
return createDefaultSource(input);
}
@SuppressWarnings("unchecked")
default Class<T> getDescriptorClass() {
default Class<T> getSourceClass() {
return (Class<T>) Arrays.stream(getClass().getDeclaredClasses())
.filter(c -> c.getName().endsWith("Descriptor")).findFirst()
.orElseThrow(() -> new AssertionError("Descriptor class not found"));
.filter(c -> c.getName().endsWith("Source")).findFirst()
.orElseThrow(() -> new ExtensionException("Descriptor class not found for " + getId()));
}
}

21
extension/src/main/java/io/xpipe/extension/DataSourceProviders.java
View file

@ -3,6 +3,7 @@ package io.xpipe.extension;
import io.xpipe.core.source.*;
import io.xpipe.core.store.DataStore;
import io.xpipe.core.store.LocalFileDataStore;
import io.xpipe.extension.event.ErrorEvent;
import lombok.SneakyThrows;
import java.util.Optional;
@ -18,7 +19,13 @@ public class DataSourceProviders {
if (ALL == null) {
ALL = ServiceLoader.load(layer, DataSourceProvider.class).stream()
.map(p -> (DataSourceProvider<?>) p.get()).collect(Collectors.toSet());
ALL.forEach(DataSourceProvider::init);
ALL.forEach(p -> {
try {
p.init();
} catch (Exception e) {
ErrorEvent.fromThrowable(e).handle();
}
});
}
}
@ -41,7 +48,7 @@ public class DataSourceProviders {
@SneakyThrows
public static StructureDataSource<LocalFileDataStore> createLocalStructureDescriptor(DataStore store) {
return (StructureDataSource<LocalFileDataStore>)
DataSourceProviders.byId("xpbs").getDescriptorClass()
DataSourceProviders.byId("xpbs").getSourceClass()
.getDeclaredConstructors()[0].newInstance(store);
}
@ -49,7 +56,7 @@ public class DataSourceProviders {
@SneakyThrows
public static RawDataSource<LocalFileDataStore> createLocalRawDescriptor(DataStore store) {
return (RawDataSource<LocalFileDataStore>)
DataSourceProviders.byId("binary").getDescriptorClass()
DataSourceProviders.byId("binary").getSourceClass()
.getDeclaredConstructors()[0].newInstance(store);
}
@ -57,7 +64,7 @@ public class DataSourceProviders {
@SneakyThrows
public static RawDataSource<LocalFileDataStore> createLocalCollectionDescriptor(DataStore store) {
return (RawDataSource<LocalFileDataStore>)
DataSourceProviders.byId("br").getDescriptorClass()
DataSourceProviders.byId("br").getSourceClass()
.getDeclaredConstructors()[0].newInstance(store);
}
@ -65,7 +72,7 @@ public class DataSourceProviders {
@SneakyThrows
public static TextDataSource<LocalFileDataStore> createLocalTextDescriptor(DataStore store) {
return (TextDataSource<LocalFileDataStore>)
DataSourceProviders.byId("text").getDescriptorClass()
DataSourceProviders.byId("text").getSourceClass()
.getDeclaredConstructors()[0].newInstance(store);
}
@ -73,7 +80,7 @@ public class DataSourceProviders {
@SneakyThrows
public static TableDataSource<LocalFileDataStore> createLocalTableDescriptor(DataStore store) {
return (TableDataSource<LocalFileDataStore>)
DataSourceProviders.byId("xpbt").getDescriptorClass()
DataSourceProviders.byId("xpbt").getSourceClass()
.getDeclaredConstructors()[0].newInstance(store);
}
@ -94,7 +101,7 @@ public class DataSourceProviders {
throw new IllegalStateException("Not initialized");
}
return (T) ALL.stream().filter(d -> d.getDescriptorClass().equals(c)).findAny()
return (T) ALL.stream().filter(d -> d.getSourceClass().equals(c)).findAny()
.orElseThrow(() -> new IllegalArgumentException("Provider for " + c.getSimpleName() + " not found"));
}

36
extension/src/main/java/io/xpipe/extension/ExtensionTest.java Normal file
View file

@ -0,0 +1,36 @@
package io.xpipe.extension;
import io.xpipe.charsetter.Charsetter;
import io.xpipe.charsetter.CharsetterContext;
import io.xpipe.core.util.JacksonHelper;
import io.xpipe.extension.util.ModuleHelper;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
public class ExtensionTest {
@BeforeAll
public static void setup() throws Exception {
var mod = ModuleLayer.boot().modules().stream()
.filter(m -> m.getName().contains(".test"))
.findFirst().orElseThrow();
var e = ModuleHelper.getEveryoneModule();
for (var pkg : mod.getPackages()) {
ModuleHelper.exportAndOpen(pkg, e);
}
var extMod = ModuleLayer.boot().modules().stream()
.filter(m -> m.getName().equals(mod.getName().replace(".test", "")))
.findFirst().orElseThrow();
for (var pkg : extMod.getPackages()) {
ModuleHelper.exportAndOpen(pkg, e);
}
JacksonHelper.initClassBased();
Charsetter.init(CharsetterContext.empty());
}
@AfterAll
public static void teardown() throws Exception {
}
}

6
extension/src/main/java/io/xpipe/extension/UniformDataSourceProvider.java
View file

@ -10,14 +10,14 @@ public interface UniformDataSourceProvider<T extends DataSource<?>> extends Data
@Override
default ConfigProvider<T> getConfigProvider() {
return ConfigProvider.empty(List.of(getId()), this::createDefaultDescriptor);
return ConfigProvider.empty(List.of(getId()), this::createDefaultSource);
}
@Override
@SuppressWarnings("unchecked")
default T createDefaultDescriptor(DataStore input) {
default T createDefaultSource(DataStore input) {
try {
return (T) getDescriptorClass().getDeclaredConstructors()[0].newInstance(input);
return (T) getSourceClass().getDeclaredConstructors()[0].newInstance(input);
} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
throw new AssertionError(e);
}

9
extension/src/main/java/io/xpipe/extension/comp/CharChoiceComp.java
View file

@ -4,21 +4,20 @@ import io.xpipe.fxcomps.Comp;
import io.xpipe.fxcomps.CompStructure;
import javafx.beans.property.Property;
import javafx.beans.property.SimpleObjectProperty;
import javafx.beans.value.ObservableValue;
import javafx.geometry.Pos;
import javafx.scene.layout.HBox;
import org.apache.commons.collections4.BidiMap;
import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
import java.util.function.Supplier;
public class CharChoiceComp extends Comp<CompStructure<HBox>> {
private final Property<Character> value;
private final Property<Character> charChoiceValue;
private final BidiMap<Character, Supplier<String>> range;
private final Supplier<String> customName;
private final BidiMap<Character, ObservableValue<String>> range;
private final ObservableValue<String> customName;
public CharChoiceComp(Property<Character> value, BidiMap<Character, Supplier<String>> range, Supplier<String> customName) {
public CharChoiceComp(Property<Character> value, BidiMap<Character, ObservableValue<String>> range, ObservableValue<String> customName) {
this.value = value;
this.range = range;
this.customName = customName;

7
extension/src/main/java/io/xpipe/extension/comp/CharsetChoiceComp.java
View file

@ -4,12 +4,13 @@ import io.xpipe.fxcomps.Comp;
import io.xpipe.fxcomps.CompStructure;
import io.xpipe.fxcomps.comp.ReplacementComp;
import javafx.beans.property.Property;
import javafx.beans.property.SimpleStringProperty;
import javafx.beans.value.ObservableValue;
import javafx.scene.control.ComboBox;
import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
import java.nio.charset.Charset;
import java.util.LinkedHashMap;
import java.util.function.Supplier;
public class CharsetChoiceComp extends ReplacementComp<CompStructure<ComboBox<Charset>>> {
@ -21,9 +22,9 @@ public class CharsetChoiceComp extends ReplacementComp<CompStructure<ComboBox<Ch
@Override
protected Comp<CompStructure<ComboBox<Charset>>> createComp() {
var map = new LinkedHashMap<Charset, Supplier<String>>();
var map = new LinkedHashMap<Charset, ObservableValue<String>>();
for (var e : Charset.availableCharsets().entrySet()) {
map.put(e.getValue(), e::getKey);
map.put(e.getValue(), new SimpleStringProperty(e.getKey()));
}
return new ChoiceComp<>(charset, new DualLinkedHashBidiMap<>(map));
}

13
extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java
View file

@ -4,19 +4,18 @@ import io.xpipe.fxcomps.Comp;
import io.xpipe.fxcomps.CompStructure;
import io.xpipe.fxcomps.util.PlatformUtil;
import javafx.beans.property.Property;
import javafx.beans.value.ObservableValue;
import javafx.collections.FXCollections;
import javafx.scene.control.ComboBox;
import javafx.util.StringConverter;
import org.apache.commons.collections4.BidiMap;
import java.util.function.Supplier;
public class ChoiceComp<T> extends Comp<CompStructure<ComboBox<T>>> {
private final Property<T> value;
private final BidiMap<T, Supplier<String>> range;
private final BidiMap<T, ObservableValue<String>> range;
public ChoiceComp(Property<T> value, BidiMap<T, Supplier<String>> range) {
public ChoiceComp(Property<T> value, BidiMap<T, ObservableValue<String>> range) {
this.value = value;
this.range = range;
}
@ -28,7 +27,11 @@ public class ChoiceComp<T> extends Comp<CompStructure<ComboBox<T>>> {
cb.setConverter(new StringConverter<>() {
@Override
public String toString(T object) {
return range.get(object).get();
if (object == null) {
return "null";
}
return range.get(object).getValue();
}
@Override

57
extension/src/main/java/io/xpipe/extension/comp/DynamicOptionsBuilder.java
View file

@ -1,17 +1,24 @@
package io.xpipe.extension.comp;
import io.xpipe.charsetter.NewLine;
import io.xpipe.core.source.DataSource;
import io.xpipe.extension.I18n;
import io.xpipe.fxcomps.Comp;
import javafx.beans.Observable;
import javafx.beans.binding.Bindings;
import javafx.beans.property.Property;
import javafx.beans.property.SimpleStringProperty;
import javafx.beans.value.ObservableValue;
import javafx.scene.control.TextField;
import javafx.scene.layout.Region;
import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.function.Supplier;
import java.util.Map;
import java.util.function.Function;
public class DynamicOptionsBuilder<T extends DataSource<?>> {
@ -22,12 +29,54 @@ public class DynamicOptionsBuilder<T extends DataSource<?>> {
var comp = new TextField();
comp.textProperty().bindBidirectional(prop);
entries.add(new DynamicOptionsComp.Entry(name, Comp.of(() -> comp)));
props.add(prop);
return this;
}
public Region build(Supplier<T> creator, Property<T> toBind) {
var bind = Bindings.createObjectBinding(() -> creator.get(), props.toArray(Observable[]::new));
toBind.bind(bind);
public DynamicOptionsBuilder<T> addNewLine(Property<NewLine> prop) {
var map = new LinkedHashMap<NewLine, ObservableValue<String>>();
for (var e : NewLine.values()) {
map.put(e, new SimpleStringProperty(e.getId()));
}
var comp = new ChoiceComp<>(prop, new DualLinkedHashBidiMap<>(map));
entries.add(new DynamicOptionsComp.Entry(I18n.observable("newLine"), comp));
props.add(prop);
return this;
}
public DynamicOptionsBuilder<T> addCharacter(Property<Character> prop, ObservableValue<String> name, Map<Character, ObservableValue<String>> names) {
var comp = new CharChoiceComp(prop, new DualLinkedHashBidiMap<>(names), null);
entries.add(new DynamicOptionsComp.Entry(name, comp));
props.add(prop);
return this;
}
public DynamicOptionsBuilder<T> addCharacter(Property<Character> prop, ObservableValue<String> name, Map<Character, ObservableValue<String>> names, ObservableValue<String> customName) {
var comp = new CharChoiceComp(prop, new DualLinkedHashBidiMap<>(names), customName);
entries.add(new DynamicOptionsComp.Entry(name, comp));
props.add(prop);
return this;
}
public <V> DynamicOptionsBuilder<T> addToggle(Property<V> prop, ObservableValue<String> name, Map<V, ObservableValue<String>> names) {
var comp = new ToggleGroupComp<>(prop, new DualLinkedHashBidiMap<>(names));
entries.add(new DynamicOptionsComp.Entry(name, comp));
props.add(prop);
return this;
}
public DynamicOptionsBuilder<T> addCharset(Property<Charset> prop) {
var comp = new CharsetChoiceComp(prop);
entries.add(new DynamicOptionsComp.Entry(I18n.observable("charset"), comp));
props.add(prop);
return this;
}
public Region build(Function<T, T> creator, Property<T> toBind) {
var bind = Bindings.createObjectBinding(() -> creator.apply(toBind.getValue()), props.toArray(Observable[]::new));
bind.addListener((c,o, n) -> {
toBind.setValue(n);
});
return new DynamicOptionsComp(entries).createRegion();
}
}

11
extension/src/main/java/io/xpipe/extension/comp/ToggleGroupComp.java
View file

@ -4,24 +4,23 @@ import io.xpipe.fxcomps.Comp;
import io.xpipe.fxcomps.CompStructure;
import io.xpipe.fxcomps.util.PlatformUtil;
import javafx.beans.property.Property;
import javafx.beans.value.ObservableValue;
import javafx.scene.control.ToggleButton;
import javafx.scene.control.ToggleGroup;
import javafx.scene.layout.HBox;
import org.apache.commons.collections4.BidiMap;
import java.util.function.Supplier;
public class ToggleGroupComp<T> extends Comp<CompStructure<HBox>> {
private final Property<T> value;
private final BidiMap<T, Supplier<String>> range;
private final BidiMap<T, ObservableValue<String>> range;
public ToggleGroupComp(Property<T> value, BidiMap<T, Supplier<String>> range) {
public ToggleGroupComp(Property<T> value, BidiMap<T, ObservableValue<String>> range) {
this.value = value;
this.range = range;
}
public BidiMap<T, Supplier<String>> getRange() {
public BidiMap<T, ObservableValue<String>> getRange() {
return range;
}
@ -31,7 +30,7 @@ public class ToggleGroupComp<T> extends Comp<CompStructure<HBox>> {
box.getStyleClass().add("toggle-group-comp");
ToggleGroup group = new ToggleGroup();
for (var entry : range.entrySet()) {
var b = new ToggleButton(entry.getValue().get());
var b = new ToggleButton(entry.getValue().getValue());
b.setOnAction(e -> {
value.setValue(entry.getKey());
e.consume();

6
extension/src/main/java/io/xpipe/extension/event/EventHandler.java
View file

@ -15,7 +15,11 @@ public abstract class EventHandler {
@Override
public void handle(TrackEvent te) {
LoggerFactory.getLogger(te.getCategory()).info(te.getMessage());
var cat = te.getCategory();
if (cat == null) {
cat = "log";
}
LoggerFactory.getLogger(cat).info(te.getMessage());
}
@Override

58
extension/src/main/java/io/xpipe/extension/util/ModuleHelper.java Normal file
View file

@ -0,0 +1,58 @@
package io.xpipe.extension.util;
import lombok.SneakyThrows;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
public class ModuleHelper {
public static void exportAndOpen(String modName) {
var mod = ModuleLayer.boot().modules().stream()
.filter(m -> m.getName().equalsIgnoreCase(modName))
.findFirst().orElseThrow();
var e = getEveryoneModule();
for (var pkg : mod.getPackages()) {
exportAndOpen(pkg, e);
}
}
@SneakyThrows
public static Module getEveryoneModule() {
Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class);
getDeclaredFields0.setAccessible(true);
Field[] fields = (Field[]) getDeclaredFields0.invoke(Module.class, false);
Field modifiers = null;
for (Field each : fields) {
if ("EVERYONE_MODULE".equals(each.getName())) {
modifiers = each;
break;
}
}
modifiers.setAccessible(true);
return (Module) modifiers.get(null);
}
@SneakyThrows
public static void exportAndOpen(String pkg, Module mod) {
if (mod.isExported(pkg) && mod.isOpen(pkg)) {
return;
}
Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredMethods0", boolean.class);
getDeclaredFields0.setAccessible(true);
Method[] fields = (Method[]) getDeclaredFields0.invoke(Module.class, false);
Method modifiers = null;
for (Method each : fields) {
if ("implAddExportsOrOpens".equals(each.getName())) {
modifiers = each;
break;
}
}
modifiers.setAccessible(true);
var e = getEveryoneModule();
modifiers.invoke(mod, pkg, e, false, true);
modifiers.invoke(mod, pkg, e, true, true);
}
}

2
extension/src/main/java/module-info.java
View file

@ -27,6 +27,8 @@ module io.xpipe.extension {
requires org.reactfx;
requires org.kordamp.ikonli.javafx;
requires com.fasterxml.jackson.databind;
requires static org.junit.jupiter.api;
requires io.xpipe.charsetter;
uses DataSourceProvider;
uses SupportedApplicationProvider;