diff --git a/.ignore b/.ignore new file mode 100644 index 0000000..26d999e --- /dev/null +++ b/.ignore @@ -0,0 +1 @@ +.devenv diff --git a/Backend/build.gradle b/Backend/build.gradle index e93dabe..60aba09 100644 --- a/Backend/build.gradle +++ b/Backend/build.gradle @@ -27,6 +27,8 @@ dependencies { implementation 'org.webjars:webjars-locator-lite' implementation 'org.webjars.npm:htmx.org:2.0.4' implementation 'org.webjars.npm:bootstrap:5.3.3' + implementation platform('software.amazon.awssdk:bom:2.45.1') + implementation 'software.amazon.awssdk:s3' runtimeOnly 'org.postgresql:postgresql' testImplementation 'org.springframework.boot:spring-boot-starter-test' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' diff --git a/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/DefaultConfig.java b/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/DefaultConfig.java index ed87db6..ca40baf 100644 --- a/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/DefaultConfig.java +++ b/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/DefaultConfig.java @@ -22,6 +22,12 @@ public class DefaultConfig { StorageService ss = new StorageService(); ss.type = StorageServiceType.LOCAL; ss.path = Path.of(System.getProperty("user.home"),".local","share","gtransfer").toString(); + ss.bucket = ""; + ss.region = "us-east-1"; + ss.endpoint = ""; + ss.accessKeyId = ""; + ss.secretAccessKey = ""; + ss.pathStyleAccessEnabled = false; c.storageService= ss; SpringConfig sc = new SpringConfig(); diff --git a/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/types/StorageService.java b/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/types/StorageService.java index a9f4cd9..f2663e8 100644 --- a/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/types/StorageService.java +++ b/Backend/src/main/java/com/gregor_lohaus/gtransfer/config/types/StorageService.java @@ -9,4 +9,16 @@ public class StorageService implements TomlSerializable { public StorageServiceType type; @Property(name = "root") public String path; + @Property(name = "bucket") + public String bucket; + @Property(name = "region") + public String region; + @Property(name = "endpoint") + public String endpoint; + @Property(name = "accessKeyId") + public String accessKeyId; + @Property(name = "secretAccessKey") + public String secretAccessKey; + @Property(name = "pathStyleAccessEnabled") + public Boolean pathStyleAccessEnabled; } diff --git a/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/S3StorageService.java b/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/S3StorageService.java new file mode 100644 index 0000000..e2683f4 --- /dev/null +++ b/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/S3StorageService.java @@ -0,0 +1,191 @@ +package com.gregor_lohaus.gtransfer.services.filewriter; + +import java.net.URI; +import java.nio.file.Path; +import java.util.Optional; +import java.util.OptionalLong; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; +import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException; +import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; +import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; + +public class S3StorageService extends AbstractStorageService { + private final S3Client client; + private final String bucket; + private final String prefix; + private final String region; + + public S3StorageService( + String bucket, + String region, + String prefix, + String endpoint, + String accessKeyId, + String secretAccessKey, + boolean pathStyleAccessEnabled) { + super(Path.of("")); + if (bucket == null || bucket.isBlank()) { + throw new IllegalArgumentException("S3 storage requires a bucket"); + } + this.bucket = bucket; + this.region = blankToDefault(region, "us-east-1"); + this.prefix = normalizePrefix(prefix); + this.client = createClient(this.region, endpoint, accessKeyId, secretAccessKey, pathStyleAccessEnabled); + ensureBucketExists(); + } + + private S3Client createClient( + String region, + String endpoint, + String accessKeyId, + String secretAccessKey, + boolean pathStyleAccessEnabled) { + S3ClientBuilder builder = S3Client.builder() + .region(Region.of(blankToDefault(region, "us-east-1"))) + .serviceConfiguration(S3Configuration.builder() + .pathStyleAccessEnabled(pathStyleAccessEnabled) + .build()); + + if (endpoint != null && !endpoint.isBlank()) { + builder.endpointOverride(URI.create(endpoint)); + } + + if (accessKeyId != null && !accessKeyId.isBlank()) { + if (secretAccessKey == null || secretAccessKey.isBlank()) { + throw new IllegalArgumentException("S3 storage requires a secretAccessKey when accessKeyId is set"); + } + builder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey))); + } else { + builder.credentialsProvider(DefaultCredentialsProvider.create()); + } + + return builder.build(); + } + + private void ensureBucketExists() { + try { + client.headBucket(HeadBucketRequest.builder() + .bucket(bucket) + .build()); + return; + } catch (S3Exception e) { + if (e.statusCode() != 404) { + throw e; + } + } + + try { + client.createBucket(createBucketRequest()); + } catch (BucketAlreadyOwnedByYouException e) { + return; + } catch (BucketAlreadyExistsException e) { + throw e; + } + } + + private CreateBucketRequest createBucketRequest() { + CreateBucketRequest.Builder builder = CreateBucketRequest.builder() + .bucket(bucket); + + if (!region.equals("us-east-1")) { + builder.createBucketConfiguration(CreateBucketConfiguration.builder() + .locationConstraint(BucketLocationConstraint.fromValue(region)) + .build()); + } + + return builder.build(); + } + + private String blankToDefault(String value, String defaultValue) { + if (value == null || value.isBlank()) { + return defaultValue; + } + return value; + } + + private String normalizePrefix(String value) { + if (value == null || value.isBlank()) { + return ""; + } + String normalized = value; + while (normalized.startsWith("/")) { + normalized = normalized.substring(1); + } + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private String key(String id) { + if (prefix.isEmpty()) { + return id; + } + return prefix + "/" + id; + } + + @Override + public OptionalLong put(String id, byte[] data) { + try { + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucket) + .key(key(id)) + .contentLength((long) data.length) + .build(); + client.putObject(request, RequestBody.fromBytes(data)); + return OptionalLong.of(data.length); + } catch (SdkException e) { + return OptionalLong.empty(); + } + } + + @Override + public Optional get(String id) { + try { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucket) + .key(key(id)) + .build(); + ResponseBytes response = client.getObjectAsBytes(request); + return Optional.of(response.asByteArray()); + } catch (NoSuchKeyException e) { + return Optional.empty(); + } catch (SdkException e) { + return Optional.empty(); + } + } + + @Override + public boolean delete(String id) { + try { + DeleteObjectRequest request = DeleteObjectRequest.builder() + .bucket(bucket) + .key(key(id)) + .build(); + client.deleteObject(request); + return true; + } catch (SdkException e) { + return false; + } + } +} diff --git a/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/StorageServiceConfiguration.java b/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/StorageServiceConfiguration.java index 6268dff..3527a20 100644 --- a/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/StorageServiceConfiguration.java +++ b/Backend/src/main/java/com/gregor_lohaus/gtransfer/services/filewriter/StorageServiceConfiguration.java @@ -11,15 +11,27 @@ import com.gregor_lohaus.gtransfer.config.types.StorageServiceType; @Configuration public class StorageServiceConfiguration { - //TODO S3 implementation @Bean public AbstractStorageService storageService( @Value("${gtransfer-config.storageService.type}") StorageServiceType type, - @Value("${gtransfer-config.storageService.root}") String root) { + @Value("${gtransfer-config.storageService.root}") String root, + @Value("${gtransfer-config.storageService.bucket:}") String bucket, + @Value("${gtransfer-config.storageService.region:us-east-1}") String region, + @Value("${gtransfer-config.storageService.endpoint:}") String endpoint, + @Value("${gtransfer-config.storageService.accessKeyId:}") String accessKeyId, + @Value("${gtransfer-config.storageService.secretAccessKey:}") String secretAccessKey, + @Value("${gtransfer-config.storageService.pathStyleAccessEnabled:false}") boolean pathStyleAccessEnabled) { return switch (type) { case LOCAL -> new LocalStorageService(Path.of(root)); case DUMMY -> new DummyStorageService(Path.of(root)); - case S3 -> new LocalStorageService(Path.of(root)); + case S3 -> new S3StorageService( + bucket, + region, + root, + endpoint, + accessKeyId, + secretAccessKey, + pathStyleAccessEnabled); }; } } diff --git a/devenv.lock b/devenv.lock index 2e5ac60..58db935 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1771852244, - "narHash": "sha256-4t3gQ4s7kqtDwSiE74Px6szKJtKtcoHgUiNmIm+Xl9Y=", + "lastModified": 1777321427, + "narHash": "sha256-EV/mIQur/dvCFwHzBjL7LBAgyhT0l3wQBgFjjY6zucg=", "owner": "cachix", "repo": "devenv", - "rev": "c88c14a32d06173867e26b7d4f5daed38a3f6f1e", + "rev": "fb3d8df47420022c47a42151c26e5cdaee6c641d", "type": "github" }, "original": { @@ -17,72 +17,16 @@ "type": "github" } }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1767039857, - "owner": "NixOS", - "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "flake-compat", - "type": "github" - } - }, - "git-hooks": { - "inputs": { - "flake-compat": "flake-compat", - "gitignore": "gitignore", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1772024342, - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "git-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1762808025, - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, "nixpkgs": { "inputs": { "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1770434727, - "narHash": "sha256-YzOZRgiqIccnkkZvckQha7wvOfN2z50xEdPvfgu6sf8=", + "lastModified": 1776852779, + "narHash": "sha256-WwO/ITisCXwyiRgtktZgv3iGhAGO+IB5Av4kKCwezR0=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348", + "rev": "ec3063523dcd911aeadb50faa589f237cdab5853", "type": "github" }, "original": { @@ -95,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1769922788, - "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=", + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", "type": "github" }, "original": { @@ -112,14 +56,10 @@ "root": { "inputs": { "devenv": "devenv", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs", - "pre-commit-hooks": [ - "git-hooks" - ] + "nixpkgs": "nixpkgs" } } }, "root": "root", "version": 7 -} +} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix index 9f74287..a474be8 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,57 +1,41 @@ { pkgs, lib, config, inputs, ... }: { - # https://devenv.sh/basics/ env.GRAALVM_HOME = "${pkgs.graalvmPackages.graalvm-ce}"; - # https://devenv.sh/packages/ packages = [ pkgs.graalvmPackages.graalvm-ce pkgs.watchexec ]; - # https://devenv.sh/languages/ languages.java.enable = true; languages.java.lsp.enable = true; languages.java.gradle.enable = true; languages.java.jdk.package = pkgs.jdk25_headless; - # https://devenv.sh/processes/ - # processes.cargo-watch.exec = "cargo-watch"; - # process.manager.implementation = "mprocs"; processes.watchbuild = { - exec = "watchexec -r -e java,html,css,js -w ./Backend/src -- build-backend"; + exec = "build-backend"; + watch = { + paths = [./Backend/src]; + extensions = ["java" "html" "css" "js"]; + }; }; processes.runbin = { - exec = "watchexec -r -w ./Backend/buildcompleted.at ./Backend/build/native/nativeCompile/gtransfer"; + exec = "./Backend/build/native/nativeCompile/gtransfer"; + watch = { + paths = [ ./Backend/buildcompleted.at ]; + }; }; - # https://devenv.sh/services/ services.postgres.enable = true; services.postgres.listen_addresses = "localhost"; services.postgres.port = 5432; services.postgres.initialDatabases = [ {name="gtransfer";user="gtransfer";pass="gtransfer";} ]; - - # https://devenv.sh/scripts/ + services.minio.enable = true; scripts.build-backend.exec = '' gradle -p ./Backend build && echo $(date) > ./Backend/buildcompleted.at ''; - # enterShell = '' - # PATH="$(pwd)/x86_64-linux-musl-native/bin:$PATH" - # ''; - # https://devenv.sh/tasks/ - # tasks = { - # "myproj:setup".exec = "mytool build"; - # "devenv:enterShell".after = [ "myproj:setup" ]; - # }; - - # https://devenv.sh/tests/ enterTest = '' ''; - - # https://devenv.sh/git-hooks/ - # git-hooks.hooks.shellcheck.enable = true; - - # See full reference at https://devenv.sh/reference/options/ }