gregor/gtransfer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

s3 support

Browse Source
This commit is contained in:
Gregor Lohaus
2026-06-05 13:51:25 +02:00
parent edf55a02c2
commit be5b9e0626
8 changed files with 248 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.ignore Normal file
View File

@@ -0,0 +1 @@
.devenv

2
Backend/build.gradle
View File

@@ -27,6 +27,8 @@ dependencies {
implementation 'org.webjars:webjars-locator-lite'
implementation 'org.webjars.npm:htmx.org:2.0.4'
implementation 'org.webjars.npm:bootstrap:5.3.3'
implementation platform('software.amazon.awssdk:bom:2.45.1')
implementation 'software.amazon.awssdk:s3'
runtimeOnly 'org.postgresql:postgresql'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'

6
Backend/src/main/java/com/gregor_lohaus/gtransfer/config/DefaultConfig.java
View File

@@ -22,6 +22,12 @@ public class DefaultConfig {
StorageService ss = new StorageService();
ss.type = StorageServiceType.LOCAL;
ss.path = Path.of(System.getProperty("user.home"),".local","share","gtransfer").toString();
ss.bucket = "";
ss.region = "us-east-1";
ss.endpoint = "";
ss.accessKeyId = "";
ss.secretAccessKey = "";
ss.pathStyleAccessEnabled = false;
c.storageService= ss;
SpringConfig sc = new SpringConfig();

12
Backend/src/main/java/com/gregor_lohaus/gtransfer/config/types/StorageService.java
View File

@@ -9,4 +9,16 @@ public class StorageService implements TomlSerializable {
public StorageServiceType type;
@Property(name = "root")
public String path;
@Property(name = "bucket")
public String bucket;
@Property(name = "region")
public String region;
@Property(name = "endpoint")
public String endpoint;
@Property(name = "accessKeyId")
public String accessKeyId;
@Property(name = "secretAccessKey")
public String secretAccessKey;
@Property(name = "pathStyleAccessEnabled")
public Boolean pathStyleAccessEnabled;
}

191
Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/S3StorageService.java Normal file
View File

@@ -0,0 +1,191 @@
package com.gregor_lohaus.gtransfer.services.filewriter;
import java.net.URI;
import java.nio.file.Path;
import java.util.Optional;
import java.util.OptionalLong;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException;
import software.amazon.awssdk.services.s3.model.BucketLocationConstraint;
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;
public class S3StorageService extends AbstractStorageService {
private final S3Client client;
private final String bucket;
private final String prefix;
private final String region;
public S3StorageService(
String bucket,
String region,
String prefix,
String endpoint,
String accessKeyId,
String secretAccessKey,
boolean pathStyleAccessEnabled) {
super(Path.of(""));
if (bucket == null || bucket.isBlank()) {
throw new IllegalArgumentException("S3 storage requires a bucket");
}
this.bucket = bucket;
this.region = blankToDefault(region, "us-east-1");
this.prefix = normalizePrefix(prefix);
this.client = createClient(this.region, endpoint, accessKeyId, secretAccessKey, pathStyleAccessEnabled);
ensureBucketExists();
}
private S3Client createClient(
String region,
String endpoint,
String accessKeyId,
String secretAccessKey,
boolean pathStyleAccessEnabled) {
S3ClientBuilder builder = S3Client.builder()
.region(Region.of(blankToDefault(region, "us-east-1")))
.serviceConfiguration(S3Configuration.builder()
.pathStyleAccessEnabled(pathStyleAccessEnabled)
.build());
if (endpoint != null && !endpoint.isBlank()) {
builder.endpointOverride(URI.create(endpoint));
}
if (accessKeyId != null && !accessKeyId.isBlank()) {
if (secretAccessKey == null || secretAccessKey.isBlank()) {
throw new IllegalArgumentException("S3 storage requires a secretAccessKey when accessKeyId is set");
}
builder.credentialsProvider(
StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey)));
} else {
builder.credentialsProvider(DefaultCredentialsProvider.create());
}
return builder.build();
}
private void ensureBucketExists() {
try {
client.headBucket(HeadBucketRequest.builder()
.bucket(bucket)
.build());
return;
} catch (S3Exception e) {
if (e.statusCode() != 404) {
throw e;
}
}
try {
client.createBucket(createBucketRequest());
} catch (BucketAlreadyOwnedByYouException e) {
return;
} catch (BucketAlreadyExistsException e) {
throw e;
}
}
private CreateBucketRequest createBucketRequest() {
CreateBucketRequest.Builder builder = CreateBucketRequest.builder()
.bucket(bucket);
if (!region.equals("us-east-1")) {
builder.createBucketConfiguration(CreateBucketConfiguration.builder()
.locationConstraint(BucketLocationConstraint.fromValue(region))
.build());
}
return builder.build();
}
private String blankToDefault(String value, String defaultValue) {
if (value == null || value.isBlank()) {
return defaultValue;
}
return value;
}
private String normalizePrefix(String value) {
if (value == null || value.isBlank()) {
return "";
}
String normalized = value;
while (normalized.startsWith("/")) {
normalized = normalized.substring(1);
}
while (normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private String key(String id) {
if (prefix.isEmpty()) {
return id;
}
return prefix + "/" + id;
}
@Override
public OptionalLong put(String id, byte[] data) {
try {
PutObjectRequest request = PutObjectRequest.builder()
.bucket(bucket)
.key(key(id))
.contentLength((long) data.length)
.build();
client.putObject(request, RequestBody.fromBytes(data));
return OptionalLong.of(data.length);
} catch (SdkException e) {
return OptionalLong.empty();
}
}
@Override
public Optional<byte[]> get(String id) {
try {
GetObjectRequest request = GetObjectRequest.builder()
.bucket(bucket)
.key(key(id))
.build();
ResponseBytes<GetObjectResponse> response = client.getObjectAsBytes(request);
return Optional.of(response.asByteArray());
} catch (NoSuchKeyException e) {
return Optional.empty();
} catch (SdkException e) {
return Optional.empty();
}
}
@Override
public boolean delete(String id) {
try {
DeleteObjectRequest request = DeleteObjectRequest.builder()
.bucket(bucket)
.key(key(id))
.build();
client.deleteObject(request);
return true;
} catch (SdkException e) {
return false;
}
}
}

18
Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/StorageServiceConfiguration.java
View File

@@ -11,15 +11,27 @@ import com.gregor_lohaus.gtransfer.config.types.StorageServiceType;
@Configuration
public class StorageServiceConfiguration {
//TODO S3 implementation
@Bean
public AbstractStorageService storageService(
@Value("${gtransfer-config.storageService.type}") StorageServiceType type,
@Value("${gtransfer-config.storageService.root}") String root) {
@Value("${gtransfer-config.storageService.root}") String root,
@Value("${gtransfer-config.storageService.bucket:}") String bucket,
@Value("${gtransfer-config.storageService.region:us-east-1}") String region,
@Value("${gtransfer-config.storageService.endpoint:}") String endpoint,
@Value("${gtransfer-config.storageService.accessKeyId:}") String accessKeyId,
@Value("${gtransfer-config.storageService.secretAccessKey:}") String secretAccessKey,
@Value("${gtransfer-config.storageService.pathStyleAccessEnabled:false}") boolean pathStyleAccessEnabled) {
return switch (type) {
case LOCAL -> new LocalStorageService(Path.of(root));
case DUMMY -> new DummyStorageService(Path.of(root));
case S3 -> new LocalStorageService(Path.of(root));
case S3 -> new S3StorageService(
bucket,
region,
root,
endpoint,
accessKeyId,
secretAccessKey,
pathStyleAccessEnabled);
};
}
}

80
devenv.lock
View File

@@ -3,11 +3,11 @@
"devenv": {
"locked": {
"dir": "src/modules",
"lastModified": 1771852244,
"narHash": "sha256-4t3gQ4s7kqtDwSiE74Px6szKJtKtcoHgUiNmIm+Xl9Y=",
"lastModified": 1777321427,
"narHash": "sha256-EV/mIQur/dvCFwHzBjL7LBAgyhT0l3wQBgFjjY6zucg=",
"owner": "cachix",
"repo": "devenv",
"rev": "c88c14a32d06173867e26b7d4f5daed38a3f6f1e",
"rev": "fb3d8df47420022c47a42151c26e5cdaee6c641d",
"type": "github"
},
"original": {
@@ -17,72 +17,16 @@
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1767039857,
"owner": "NixOS",
"repo": "flake-compat",
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "flake-compat",
"type": "github"
}
},
"git-hooks": {
"inputs": {
"flake-compat": "flake-compat",
"gitignore": "gitignore",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1772024342,
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"git-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1762808025,
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixpkgs": {
"inputs": {
"nixpkgs-src": "nixpkgs-src"
},
"locked": {
"lastModified": 1770434727,
"narHash": "sha256-YzOZRgiqIccnkkZvckQha7wvOfN2z50xEdPvfgu6sf8=",
"lastModified": 1776852779,
"narHash": "sha256-WwO/ITisCXwyiRgtktZgv3iGhAGO+IB5Av4kKCwezR0=",
"owner": "cachix",
"repo": "devenv-nixpkgs",
"rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348",
"rev": "ec3063523dcd911aeadb50faa589f237cdab5853",
"type": "github"
},
"original": {
@@ -95,11 +39,11 @@
"nixpkgs-src": {
"flake": false,
"locked": {
"lastModified": 1769922788,
"narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=",
"lastModified": 1776329215,
"narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "207d15f1a6603226e1e223dc79ac29c7846da32e",
"rev": "b86751bc4085f48661017fa226dee99fab6c651b",
"type": "github"
},
"original": {
@@ -112,11 +56,7 @@
"root": {
"inputs": {
"devenv": "devenv",
"git-hooks": "git-hooks",
"nixpkgs": "nixpkgs",
"pre-commit-hooks": [
"git-hooks"
]
"nixpkgs": "nixpkgs"
}
}
},

36
devenv.nix
View File

@@ -1,57 +1,41 @@
{ pkgs, lib, config, inputs, ... }:
{
# https://devenv.sh/basics/
env.GRAALVM_HOME = "${pkgs.graalvmPackages.graalvm-ce}";
# https://devenv.sh/packages/
packages = [
pkgs.graalvmPackages.graalvm-ce
pkgs.watchexec
];
# https://devenv.sh/languages/
languages.java.enable = true;
languages.java.lsp.enable = true;
languages.java.gradle.enable = true;
languages.java.jdk.package = pkgs.jdk25_headless;
# https://devenv.sh/processes/
# processes.cargo-watch.exec = "cargo-watch";
# process.manager.implementation = "mprocs";
processes.watchbuild = {
exec = "watchexec -r -e java,html,css,js -w ./Backend/src -- build-backend";
exec = "build-backend";
watch = {
paths = [./Backend/src];
extensions = ["java" "html" "css" "js"];
};
};
processes.runbin = {
exec = "watchexec -r -w ./Backend/buildcompleted.at ./Backend/build/native/nativeCompile/gtransfer";
exec = "./Backend/build/native/nativeCompile/gtransfer";
watch = {
paths = [ ./Backend/buildcompleted.at ];
};
};
# https://devenv.sh/services/
services.postgres.enable = true;
services.postgres.listen_addresses = "localhost";
services.postgres.port = 5432;
services.postgres.initialDatabases = [
{name="gtransfer";user="gtransfer";pass="gtransfer";}
];
# https://devenv.sh/scripts/
services.minio.enable = true;
scripts.build-backend.exec = ''
gradle -p ./Backend build && echo $(date) > ./Backend/buildcompleted.at
'';
# enterShell = ''
# PATH="$(pwd)/x86_64-linux-musl-native/bin:$PATH"
# '';
# https://devenv.sh/tasks/
# tasks = {
# "myproj:setup".exec = "mytool build";
# "devenv:enterShell".after = [ "myproj:setup" ];
# };
# https://devenv.sh/tests/
enterTest = ''
'';
# https://devenv.sh/git-hooks/
# git-hooks.hooks.shellcheck.enable = true;
# See full reference at https://devenv.sh/reference/options/
}