Remove supportlib

nice idea, but ram usage explosion seems to cancel out the benefits
This commit is contained in:
2025-04-19 11:12:04 +02:00
parent 59e8f6a6b4
commit e6f95ef028
40 changed files with 7 additions and 833 deletions

View File

@@ -2,7 +2,7 @@
<configuration default="false" name="Main 2" type="QsApplicationConfigurationType" factoryName="QuarkusApplication">
<option name="MAIN_CLASS_NAME" value="com.usatiuk.dhfs.app.Main" />
<module name="dhfs-app" />
<option name="VM_PARAMETERS" value="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Dcom.usatiuk.dhfs.supportlib.native-path=$ProjectFileDir$/target/classes/native -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/2/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/2/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/2/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=false -Dquarkus.http.port=9020 -Dquarkus.http.ssl-port=9021 -Ddhfs.peerdiscovery.preset-uuid=22000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=11000000-0000-0000-0000-000000000000:127.0.0.1:9010:9011" />
<option name="VM_PARAMETERS" value="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/2/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/2/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/2/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=false -Dquarkus.http.port=9020 -Dquarkus.http.ssl-port=9021 -Ddhfs.peerdiscovery.preset-uuid=22000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=11000000-0000-0000-0000-000000000000:127.0.0.1:8080:9011" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.usatiuk.dhfs.*" />

View File

@@ -2,7 +2,7 @@
<configuration default="false" name="Main" type="QsApplicationConfigurationType" factoryName="QuarkusApplication" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.usatiuk.dhfs.app.Main" />
<module name="dhfs-app" />
<option name="VM_PARAMETERS" value="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Dcom.usatiuk.dhfs.supportlib.native-path=$ProjectFileDir$/target/classes/native -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/1/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/1/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/1/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=true -Dquarkus.http.port=8080 -Dquarkus.http.ssl-port=9011 -Ddhfs.peerdiscovery.preset-uuid=11000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=22000000-0000-0000-0000-000000000000:127.0.0.1:9020:9021 -Dquarkus.http.host=0.0.0.0" />
<option name="VM_PARAMETERS" value="-XX:+UnlockDiagnosticVMOptions --enable-preview -XX:+UseParallelGC -XX:+DebugNonSafepoints --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/1/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/1/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/1/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=true -Dquarkus.http.port=8080 -Dquarkus.http.ssl-port=9011 -Ddhfs.peerdiscovery.preset-uuid=11000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=22000000-0000-0000-0000-000000000000:127.0.0.1:9020:9021 -Dquarkus.http.host=0.0.0.0" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.usatiuk.dhfs.*" />

View File

@@ -127,11 +127,6 @@
<artifactId>kleppmanntree</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>supportlib</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>objects</artifactId>

View File

@@ -79,7 +79,6 @@ public class DhfsImage implements Future<String> {
"-Ddhfs.objects.sync.timeout=10",
"-Ddhfs.objects.sync.ping.timeout=5",
"-Ddhfs.objects.reconnect_interval=1s",
"-Dcom.usatiuk.dhfs.supportlib.native-path=/libs",
"-Dquarkus.log.category.\"com.usatiuk\".level=TRACE",
"-Dquarkus.log.category.\"com.usatiuk.dhfs\".level=TRACE",
"-Ddhfs.objects.periodic-push-op-interval=5s",

View File

@@ -5,7 +5,6 @@ dhfs.objects.ref_verification=true
dhfs.objects.deletion.delay=0
quarkus.log.category."com.usatiuk.dhfs".level=TRACE
quarkus.log.category."com.usatiuk.dhfs".min-level=TRACE
quarkus.class-loading.parent-first-artifacts=com.usatiuk.dhfs:supportlib
quarkus.http.test-port=0
quarkus.http.test-ssl-port=0
dhfs.local-discovery=false

View File

@@ -127,11 +127,6 @@
<artifactId>kleppmanntree</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>supportlib</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>objects</artifactId>

View File

@@ -5,7 +5,6 @@ dhfs.objects.ref_verification=true
dhfs.objects.deletion.delay=0
quarkus.log.category."com.usatiuk.dhfs".level=TRACE
quarkus.log.category."com.usatiuk.dhfs".min-level=TRACE
quarkus.class-loading.parent-first-artifacts=com.usatiuk.dhfs:supportlib
quarkus.http.test-port=0
quarkus.http.test-ssl-port=0
dhfs.local-discovery=false

View File

@@ -127,11 +127,6 @@
<artifactId>kleppmanntree</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>supportlib</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>objects</artifactId>

View File

@@ -5,7 +5,6 @@ import com.sun.security.auth.module.UnixSystem;
import com.usatiuk.dhfs.files.service.DhfsFileService;
import com.usatiuk.dhfs.files.service.DirectoryNotEmptyException;
import com.usatiuk.dhfs.files.service.GetattrRes;
import com.usatiuk.dhfs.supportlib.UninitializedByteBuffer;
import com.usatiuk.kleppmanntree.AlreadyExistsException;
import com.usatiuk.objects.JObjectKey;
import io.grpc.Status;
@@ -28,6 +27,7 @@ import ru.serce.jnrfuse.struct.FuseFileInfo;
import ru.serce.jnrfuse.struct.Statvfs;
import ru.serce.jnrfuse.struct.Timespec;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Optional;
@@ -237,7 +237,7 @@ public class DhfsFuse extends FuseStubFS {
if (offset < 0) return -ErrorCodes.EINVAL();
try {
var fileKey = getFromHandle(fi.fh.get());
var buffer = UninitializedByteBuffer.allocateUninitialized((int) size);
var buffer = ByteBuffer.allocateDirect((int) size);
if (buffer.isDirect()) {
jnrPtrByteOutputAccessors.getUnsafe().copyMemory(

View File

@@ -5,7 +5,6 @@ dhfs.objects.ref_verification=true
dhfs.objects.deletion.delay=0
quarkus.log.category."com.usatiuk.dhfs".level=TRACE
quarkus.log.category."com.usatiuk.dhfs".min-level=TRACE
quarkus.class-loading.parent-first-artifacts=com.usatiuk.dhfs:supportlib
quarkus.http.test-port=0
quarkus.http.test-ssl-port=0
dhfs.local-discovery=false

View File

@@ -59,11 +59,6 @@
<artifactId>utils</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>supportlib</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-mockito</artifactId>

View File

@@ -1,7 +1,5 @@
package com.usatiuk.objects;
import com.usatiuk.dhfs.supportlib.UninitializedByteBuffer;
import java.io.Serial;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -48,7 +46,7 @@ public final class JObjectKeyImpl implements JObjectKey {
synchronized (this) {
if (_bb != null) return _bb;
var bytes = value.getBytes(StandardCharsets.ISO_8859_1);
var directBb = UninitializedByteBuffer.allocateUninitialized(bytes.length);
var directBb = ByteBuffer.allocateDirect(bytes.length);
directBb.put(bytes);
directBb.flip();
_bb = directBb;

View File

@@ -16,7 +16,6 @@
<module>dhfs-fuse</module>
<module>dhfs-app</module>
<module>kleppmanntree</module>
<module>supportlib</module>
<module>objects</module>
<module>utils</module>
</modules>
@@ -122,9 +121,6 @@
<systemPropertyVariables>
<java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager>
<maven.home>${maven.home}</maven.home>
<com.usatiuk.dhfs.supportlib.native-path>
${dhfs.native-libs-dir}
</com.usatiuk.dhfs.supportlib.native-path>
</systemPropertyVariables>
<argLine>
--add-exports java.base/sun.nio.ch=ALL-UNNAMED
@@ -192,7 +188,6 @@
--initialize-at-run-time=com.usatiuk.dhfs.utils.DataLocker$Lock,
--initialize-at-run-time=com.usatiuk.objects.stores.LmdbObjectPersistentStore$LmdbKvIterator,
--initialize-at-run-time=com.usatiuk.objects.stores.LmdbObjectPersistentStore,
--initialize-at-run-time=com.usatiuk.dhfs.supportlib.UninitializedByteBuffer,
--initialize-at-run-time=com.google.protobuf.UnsafeUtil
</quarkus.native.additional-build-args>
</properties>

View File

@@ -1,114 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>parent</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>
<artifactId>supportlib</artifactId>
<properties>
<cmake.download>false</cmake.download>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId>
<version>3.4.1</version>
<executions>
<execution>
<id>CMake Configure</id>
<phase>generate-sources</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>
${project.parent.basedir}/../libdhfs_support/builder/cross-build.sh
</executable>
<arguments>
<argument>configure</argument>
<argument>${project.build.outputDirectory}/native-build</argument>
<argument>${dhfs.native-libs-dir}</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>CMake Build</id>
<phase>compile</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>
${project.parent.basedir}/../libdhfs_support/builder/cross-build.sh
</executable>
<arguments>
<argument>build</argument>
<argument>${project.build.outputDirectory}/native-build</argument>
<argument>${dhfs.native-libs-dir}</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.googlecode.cmake-maven-project</groupId>
<artifactId>cmake-maven-plugin</artifactId>
<version>3.30.2-b1</version>
<executions>
<execution>
<id>cmake-generate</id>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<sourcePath>
${project.parent.basedir}/../libdhfs_support
</sourcePath>
<targetPath>
${project.build.outputDirectory}/native-build-local
</targetPath>
<options>
<option>
-DJAVA_HOME=${java.home}
</option>
<option>
-DDHFS_LIB_INSTALL=${dhfs.native-libs-dir}
</option>
</options>
</configuration>
</execution>
<execution>
<id>cmake-compile</id>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<target>
install
</target>
<projectDirectory>
${project.build.outputDirectory}/native-build-local
</projectDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,15 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.nio.file.Path;
class DhfsNativeLibFinder {
static private final String LibName = "libdhfs_support";
static Path getLibPath() {
var override = System.getProperty("com.usatiuk.dhfs.supportlib.native-path-override");
if (override != null)
return Path.of(override);
return Path.of(System.getProperty("com.usatiuk.dhfs.supportlib.native-path"))
.resolve(SysUtils.getLibPlatform() + "-" + SysUtils.getLibArch()).resolve(LibName + "." + SysUtils.getLibExtension());
}
}

View File

@@ -1,35 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.nio.ByteBuffer;
import java.util.logging.Logger;
public class DhfsSupport {
public static final int PAGE_SIZE;
private static final Logger LOGGER = Logger.getLogger(DhfsSupport.class.getName());
private static final DhfsSupportImpl IMPLEMENTATION;
static {
DhfsSupportImpl tmp;
try {
System.load(DhfsNativeLibFinder.getLibPath().toAbsolutePath().toString());
tmp = new DhfsSupportImplNative();
} catch (Throwable e) {
LOGGER.warning("Failed to load native libraries, using fallback: \n" + e);
tmp = new DhfsSupportImplFallback();
}
IMPLEMENTATION = tmp;
PAGE_SIZE = getPageSizeInternal();
}
static long allocateUninitializedByteBuffer(ByteBuffer[] bb, int size) {
return IMPLEMENTATION.allocateUninitializedByteBuffer(bb, size);
}
static void releaseByteBuffer(long token) {
IMPLEMENTATION.releaseByteBuffer(token);
}
private static int getPageSizeInternal() {
return IMPLEMENTATION.getPageSizeInternal();
}
}

View File

@@ -1,11 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.nio.ByteBuffer;
interface DhfsSupportImpl {
long allocateUninitializedByteBuffer(ByteBuffer[] bb, int size);
void releaseByteBuffer(long token);
int getPageSizeInternal();
}

View File

@@ -1,21 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.nio.ByteBuffer;
class DhfsSupportImplFallback implements DhfsSupportImpl {
@Override
public long allocateUninitializedByteBuffer(ByteBuffer[] bb, int size) {
bb[0] = ByteBuffer.allocateDirect(size);
return -1;
}
@Override
public void releaseByteBuffer(long token) {
// GC
}
@Override
public int getPageSizeInternal() {
return 4096; // FIXME:?
}
}

View File

@@ -1,20 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.nio.ByteBuffer;
class DhfsSupportImplNative implements DhfsSupportImpl {
@Override
public long allocateUninitializedByteBuffer(ByteBuffer[] bb, int size) {
return DhfsSupportNative.allocateUninitializedByteBuffer(bb, size);
}
@Override
public void releaseByteBuffer(long token) {
DhfsSupportNative.releaseByteBuffer(token);
}
@Override
public int getPageSizeInternal() {
return DhfsSupportNative.PAGE_SIZE;
}
}

View File

@@ -1,20 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.nio.ByteBuffer;
class DhfsSupportNative {
static public final int PAGE_SIZE;
static {
System.load(DhfsNativeLibFinder.getLibPath().toAbsolutePath().toString());
PAGE_SIZE = getPageSizeInternal();
}
static native long allocateUninitializedByteBuffer(ByteBuffer[] bb, int size);
static native void releaseByteBuffer(long token);
private static native int getPageSizeInternal();
}

View File

@@ -1,43 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import org.apache.commons.lang3.SystemUtils;
class SysUtils {
static String getLibPlatform() {
if (SystemUtils.IS_OS_MAC) {
return "Darwin";
} else if (SystemUtils.IS_OS_LINUX) {
return "Linux";
} else {
throw new IllegalStateException("Unsupported OS: " + SystemUtils.OS_NAME);
}
}
static String getLibExtension() {
if (SystemUtils.IS_OS_MAC) {
return "dylib";
} else if (SystemUtils.IS_OS_LINUX) {
return "so";
} else {
throw new IllegalStateException("Unsupported OS: " + SystemUtils.OS_NAME);
}
}
static String getLibArch() {
if (SystemUtils.IS_OS_MAC) {
return switch (SystemUtils.OS_ARCH) {
case "aarch64" -> "arm64";
default -> throw new IllegalStateException("Unsupported architecture: " + SystemUtils.OS_ARCH);
};
} else if (SystemUtils.IS_OS_LINUX) {
return switch (SystemUtils.OS_ARCH) {
case "aarch64" -> "aarch64";
case "amd64" -> "x86_64";
default -> throw new IllegalStateException("Unsupported architecture: " + SystemUtils.OS_ARCH);
};
} else {
throw new IllegalStateException("Unsupported OS: " + SystemUtils.OS_NAME);
}
}
}

View File

@@ -1,28 +0,0 @@
package com.usatiuk.dhfs.supportlib;
import java.lang.ref.Cleaner;
import java.nio.ByteBuffer;
import java.util.logging.Logger;
public class UninitializedByteBuffer {
private static final Cleaner CLEANER = Cleaner.create();
private static final Logger LOGGER = Logger.getLogger(UninitializedByteBuffer.class.getName());
public static ByteBuffer allocateUninitialized(int size) {
if (size < DhfsSupport.PAGE_SIZE)
return ByteBuffer.allocateDirect(size);
var bb = new ByteBuffer[1];
long token = DhfsSupport.allocateUninitializedByteBuffer(bb, size);
var ret = bb[0];
CLEANER.register(ret, () -> {
try {
DhfsSupport.releaseByteBuffer(token);
} catch (Throwable e) {
LOGGER.severe("Error releasing buffer: " + e);
System.exit(-1);
}
});
return ret;
}
}

View File

@@ -127,11 +127,6 @@
<artifactId>kleppmanntree</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>supportlib</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.usatiuk.dhfs</groupId>
<artifactId>objects</artifactId>

View File

@@ -5,7 +5,6 @@ dhfs.objects.ref_verification=true
dhfs.objects.deletion.delay=0
quarkus.log.category."com.usatiuk.dhfs".level=TRACE
quarkus.log.category."com.usatiuk.dhfs".min-level=TRACE
quarkus.class-loading.parent-first-artifacts=com.usatiuk.dhfs:supportlib
quarkus.http.test-port=0
quarkus.http.test-ssl-port=0
dhfs.local-discovery=false