From 48151e1c39169e92e140fa0f1836baada0192996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C4=9Bnceslav=20Chumchal?= Date: Tue, 29 Jul 2025 12:07:00 +0200 Subject: [PATCH 1/5] Resolve "Simplify configuration one file" --- .gitignore | 2 +- README.md | 40 +++-- build-release/Dockerfile | 8 +- build-release/build.bat | 4 +- build-release/build.sh | 4 +- build-release/release.bat | 4 +- build-release/release.sh | 4 +- deploy/{.env => .env.tofill} | 27 ++- deploy/app-keys/ddrcore.p12.toput | 1 + deploy/config/application.properties | 18 +- deploy/config/vault.hcl | 6 +- deploy/docker-compose.yaml | 29 ++-- deploy/keystore/s3core.p12 | 0 deploy/ssl/s3core.crt | 0 deploy/ssl/s3corewp.key | 0 deploy/vault-keys/ddrcore.crt.toput | 1 + deploy/vault-keys/ddrcore_decrypted.key.toput | 1 + develop/.env.tofill | 34 ++++ develop/config/application.properties | 10 +- develop/docker-compose.yaml | 19 +-- develop/fill.env | 12 -- settings.gradle | 2 +- .../tul/cxi/DDRcore/DDRcoreApplication.java | 12 ++ .../DDRcore/component/ApiKeyAuthFilter.java | 43 +++++ .../component/JSONComponent.java | 4 +- .../DDRcore/component/S3ClientProvider.java | 94 ++++++++++ .../component/S3Component.java | 59 +++---- .../config/SecurityConfig.java | 22 ++- .../config/SecurityTestConfig.java | 15 +- .../tul/cxi/DDRcore/config/VaultConfig.java | 34 ++++ .../config/VaultTestConfig.java | 27 ++- .../controller/RestController.java | 41 ++--- .../CredentialsNotFoundException.java | 7 + .../exception/DDRcoreExceptionHandler.java | 28 +++ .../MissingAdminEinfraIdException.java | 7 + .../S3ClientInitializationException.java | 7 + .../tul/cxi/DDRcore/model/BucketPolicy.java | 11 ++ .../cz/tul/cxi/DDRcore/model/Credentials.java | 22 +++ .../model/S3Bucket.java | 6 +- .../cz/tul/cxi/DDRcore/model/S3Object.java | 45 +++++ .../cz/tul/cxi/DDRcore/model/Statement.java | 12 ++ .../DDRcore/service/CredentialsService.java | 83 +++++++++ .../service/S3Service.java | 161 ++++++++++-------- .../S3PointCore/S3PointCoreApplication.java | 13 -- .../component/ApiKeyAuthFilter.java | 39 ----- .../component/S3ClientProvider.java | 91 ---------- .../cxi/S3PointCore/config/VaultConfig.java | 37 ---- .../CredentialsNotFoundException.java | 7 - .../MissingAdminEinfraIdException.java | 7 - .../S3ClientInitializationException.java | 7 - .../exception/S3PointExceptionHandler.java | 35 ---- .../cxi/S3PointCore/model/BucketPolicy.java | 12 -- .../cxi/S3PointCore/model/Credentials.java | 20 --- .../tul/cxi/S3PointCore/model/S3Object.java | 45 ----- .../tul/cxi/S3PointCore/model/Statement.java | 12 -- .../service/CredentialsService.java | 75 -------- ...itional-spring-configuration-metadata.json | 48 ++++++ .../cxi/DDRcore/DDRcoreApplicationTests.java | 13 ++ .../cxi/DDRcore/TestDDRcoreApplication.java | 12 ++ .../S3PointCoreApplicationTests.java | 18 -- .../TestS3PointCoreApplication.java | 13 -- src/test/resources/application.properties | 2 +- 62 files changed, 777 insertions(+), 695 deletions(-) rename deploy/{.env => .env.tofill} (58%) create mode 100644 deploy/app-keys/ddrcore.p12.toput delete mode 100644 deploy/keystore/s3core.p12 delete mode 100644 deploy/ssl/s3core.crt delete mode 100644 deploy/ssl/s3corewp.key create mode 100644 deploy/vault-keys/ddrcore.crt.toput create mode 100644 deploy/vault-keys/ddrcore_decrypted.key.toput create mode 100644 develop/.env.tofill delete mode 100644 develop/fill.env create mode 100644 src/main/java/cz/tul/cxi/DDRcore/DDRcoreApplication.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/component/ApiKeyAuthFilter.java rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/component/JSONComponent.java (95%) create mode 100644 src/main/java/cz/tul/cxi/DDRcore/component/S3ClientProvider.java rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/component/S3Component.java (90%) rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/config/SecurityConfig.java (59%) rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/config/SecurityTestConfig.java (65%) create mode 100644 src/main/java/cz/tul/cxi/DDRcore/config/VaultConfig.java rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/config/VaultTestConfig.java (54%) rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/controller/RestController.java (91%) create mode 100644 src/main/java/cz/tul/cxi/DDRcore/exception/CredentialsNotFoundException.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/exception/DDRcoreExceptionHandler.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/exception/MissingAdminEinfraIdException.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/exception/S3ClientInitializationException.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/model/BucketPolicy.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/model/Credentials.java rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/model/S3Bucket.java (66%) create mode 100644 src/main/java/cz/tul/cxi/DDRcore/model/S3Object.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/model/Statement.java create mode 100644 src/main/java/cz/tul/cxi/DDRcore/service/CredentialsService.java rename src/main/java/cz/tul/cxi/{S3PointCore => DDRcore}/service/S3Service.java (74%) delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/S3PointCoreApplication.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/component/ApiKeyAuthFilter.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/component/S3ClientProvider.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/config/VaultConfig.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/exception/CredentialsNotFoundException.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/exception/MissingAdminEinfraIdException.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/exception/S3ClientInitializationException.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/exception/S3PointExceptionHandler.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/model/BucketPolicy.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/model/Credentials.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/model/S3Object.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/model/Statement.java delete mode 100644 src/main/java/cz/tul/cxi/S3PointCore/service/CredentialsService.java create mode 100644 src/main/resources/META-INF/additional-spring-configuration-metadata.json create mode 100644 src/test/java/cz/tul/cxi/DDRcore/DDRcoreApplicationTests.java create mode 100644 src/test/java/cz/tul/cxi/DDRcore/TestDDRcoreApplication.java delete mode 100644 src/test/java/cz/tul/cxi/S3PointCore/S3PointCoreApplicationTests.java delete mode 100644 src/test/java/cz/tul/cxi/S3PointCore/TestS3PointCoreApplication.java diff --git a/.gitignore b/.gitignore index 5a58be4..95877e9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ build/ !gradle/wrapper/gradle-wrapper.jar !**/src/main/**/build/ !**/src/test/**/build/ -S3PointCore.jar +DDRcore.jar ### STS ### .apt_generated diff --git a/README.md b/README.md index 7145bdd..f506ba0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# S3Point Core - a core services for S3Point browser +# DDRcore - a core services for DDRplatform ## Description @@ -9,7 +9,7 @@ Docker Compose. ## Deployment Guide -This guide provides detailed instructions for deploying the S3Point Core web service with Docker Compose and configuring HashiCorp Vault for secure data management. +This guide provides detailed instructions for deploying the DDRcore web service with Docker Compose and configuring HashiCorp Vault for secure data management. --- @@ -18,19 +18,20 @@ This guide provides detailed instructions for deploying the S3Point Core web ser 1. **Download and Arrange Deployment Files**: Begin by downloading the `deploy` directory to your target server. This directory should contain: - **docker-compose.yaml**: Defines Docker Compose configuration. - - **application.properties**: Configures application-specific settings for S3Point Core. + - **application.properties**: Configures application-specific settings for DDRcore. - **.env**: Contains environment variables for Docker and the application, including tokens and credentials. - - **keystore and ssl Directories**: Place public/private keys and certificates here. - - **keystore**: Stores keys in Java format. - - **ssl**: Stores certificates and keys in PEM format. + - **app-keys and vault-keys directories**: Place public/private keys and certificates here. + - **app-keys**: Stores keys in Java format. + - **vault-keys**: Stores certificates and keys in PEM format. 2. **S3 and Vault Access Configuration**: - In `docker-compose.yaml`, `application.properties`, and `.env`, locate the placeholder values (marked as `<<>>`) and replace them with: + In `.env`, locate the placeholder values (marked as `<<>>`) and replace them with: - **Vault access**: For securely storing and retrieving sensitive application data. - **S3 access**: Required for backend access to Amazon S3. + - Other requested deployment information. 3. **Set Certificates and Keys**: - Ensure the paths to certificates and keys are correctly set in the `keystore` and `ssl` directories. + Ensure the paths to certificates and keys are correctly set in the `app-keys` and `vault-keys` directories. --- @@ -42,13 +43,15 @@ This guide provides detailed instructions for deploying the S3Point Core web ser docker compose up --wait -This will initialize the S3Point Core and Vault containers. The `--wait` flag ensures Docker waits until all containers are fully up and running. +This will initialize the DDRcore and Vault containers. The `--wait` flag ensures Docker waits until all containers are fully up and running. ### 2. Vault Configuration (First-time Setup): 1. Open the Vault GUI by navigating to `https://your-domain.com:8201`. -2. Unseal Vault: -Vault is initially sealed for security. To unseal: +2. Initiliaze Vault: +Vault is initially sealed for security. To intialize and unseal: +- Generate unseal keys. +- Download generated keys and root token as JSON. - Enter the unseal keys generated during initialization (multiple keys may be required). - Store these keys securely for future use, as they are needed if Vault is restarted. @@ -59,7 +62,7 @@ After unsealing, Vault issues a root token for initial access. Store this token - Create a Key-Value (KV) secrets engine called `kv` (see Vault KV documentation). - Populate the database with required credentials and secrets: - **e-infra ID** (record name): The ID associated with your infrastructure. - - **accessKey** and **secretKey** (key-value pairs): Corresponding S3 credentials. + - **access_key** and **secret_key** (key-value pairs): Corresponding S3 credentials. ## 3. Restart Application with Vault Token @@ -67,11 +70,12 @@ After unsealing, Vault issues a root token for initial access. Store this token Copy the root token obtained in step 2 into the `.env` file under `VAULT_TOKEN`. 2. Restart the Web Service Container: -- Terminate the existing S3Point Core container: +- Terminate the existing DDRcore container and restart: ```bash - docker kill s3point-core-1 + docker kill ddr-core-1 + docker compose up --wait -This restart initializes the application with access to Vault, allowing S3Point Core to read secrets from the KV database. +This restart initializes the application with access to Vault, allowing DDRcore to read secrets from the KV database. Maintenance Note: Unsealing Vault After Updates Vault Auto-Unsealing: @@ -89,12 +93,12 @@ Then for local develoment instance run ./local-deploy.sh `` -When running S3Point Core locally with vault note that it's in dev mode and dev mode does not support persistent storage – all data (secrets, policies, tokens, etc.) are stored in memory only and will be lost when the container restarts. +When running DDRcore locally with vault note that it's in dev mode and dev mode does not support persistent storage – all data (secrets, policies, tokens, etc.) are stored in memory only and will be lost when the container restarts. **After each start:** Access the container using: `` -docker exec -it s3point-test2-vault-1 sh +docker exec -it ddr-test2-vault-1 sh `` Set the `VAULT_ADDR` environment variable manually: @@ -126,7 +130,7 @@ Note: The target versions used by our developer team are **production** and **stage**. The container registry is used to release -on [Docker Hub](https://hub.docker.com/r/cxiomi/s3point-core) with the hub name **cxiomi**, where the +on [Docker Hub](https://hub.docker.com/r/cxiomi/ddr-core) with the hub name **cxiomi**, where the images are uploaded for subsequent deployment. The application is developed so that the actual deployment can be done with only a change of configuration files in the *deploy* directory. In the case of modifying the code and building an image for your institution, you need to use your container repository. diff --git a/build-release/Dockerfile b/build-release/Dockerfile index 4cc4b2e..862aaa1 100755 --- a/build-release/Dockerfile +++ b/build-release/Dockerfile @@ -11,10 +11,10 @@ COPY "truststore/GEANTOVRSACA4.cer" "/tmp/GEANTOVRSACA4.cer" COPY "truststore/HARICATLSRSAROOT2021.cer" "/tmp/HARICATLSRSAROOT2021.cer" RUN cd $JAVA_HOME/lib/security \ - && keytool -importcert -storepass changeit -noprompt -cacerts -alias GEANTTLSRSA1 -file /tmp/GEANTTLSRSA1.cer \ - && keytool -importcert -storepass changeit -noprompt -cacerts -alias GEANTOVRSACA4 -file /tmp/GEANTOVRSACA4.cer \ - && keytool -importcert -storepass changeit -noprompt -cacerts -alias HARICATLSRSAROOT2021 -file /tmp/HARICATLSRSAROOT2021.cer + && keytool -importcert -storepass changeit -noprompt -cacerts -alias GEANTTLSRSA1 -file /tmp/GEANTTLSRSA1.cer \ + && keytool -importcert -storepass changeit -noprompt -cacerts -alias GEANTOVRSACA4 -file /tmp/GEANTOVRSACA4.cer \ + && keytool -importcert -storepass changeit -noprompt -cacerts -alias HARICATLSRSAROOT2021 -file /tmp/HARICATLSRSAROOT2021.cer -COPY "$TARGET_FILE" "/app/S3PointCore.jar" +COPY "$TARGET_FILE" "/app/DDRcore.jar" WORKDIR "/app" diff --git a/build-release/build.bat b/build-release/build.bat index 4d63619..493a257 100755 --- a/build-release/build.bat +++ b/build-release/build.bat @@ -1,5 +1,5 @@ @echo off set VERSION=%1 -set FILE=S3PointCore-%VERSION%.jar +set FILE=DDRcore-%VERSION%.jar -gradlew build -Pversion="%VERSION%" && copy "build\libs\%FILE%" "S3PointCore.jar" \ No newline at end of file +gradlew build -Pversion="%VERSION%" && copy "build\libs\%FILE%" "DDRcore.jar" \ No newline at end of file diff --git a/build-release/build.sh b/build-release/build.sh index 6fca689..93da7c1 100755 --- a/build-release/build.sh +++ b/build-release/build.sh @@ -1,7 +1,7 @@ #!/bin/bash VERSION=$1 -FILE=S3PointCore-$VERSION.jar +FILE=DDRcore-$VERSION.jar ./gradlew build -Pversion="$VERSION" -cp "build/libs/$FILE" "S3PointCore.jar" +cp "build/libs/$FILE" "DDRcore.jar" diff --git a/build-release/release.bat b/build-release/release.bat index 9bd4a39..7fb0de9 100644 --- a/build-release/release.bat +++ b/build-release/release.bat @@ -2,8 +2,8 @@ set HUB=%1 set VERSION=%2 -set TARGET_FILE=S3PointCore.jar -set FULL_IMAGE_NAME=%HUB%/s3point-core:%VERSION% +set TARGET_FILE=DDRcore.jar +set FULL_IMAGE_NAME=%HUB%/ddr-core:%VERSION% echo %PWD% diff --git a/build-release/release.sh b/build-release/release.sh index 8543b04..51ffbdf 100755 --- a/build-release/release.sh +++ b/build-release/release.sh @@ -4,8 +4,8 @@ export DOCKER_DEFAULT_PLATFORM=linux/amd64 HUB=$1 VERSION=$2 -TARGET_FILE=S3PointCore.jar -FULL_IMAGE_NAME="$HUB/s3point-core:$VERSION" +TARGET_FILE=DDRcore.jar +FULL_IMAGE_NAME="$HUB/ddr-core:$VERSION" echo "$PWD" diff --git a/deploy/.env b/deploy/.env.tofill similarity index 58% rename from deploy/.env rename to deploy/.env.tofill index 419c5c1..1964556 100644 --- a/deploy/.env +++ b/deploy/.env.tofill @@ -1,31 +1,44 @@ # This file is for setting secrets and confidential information # For the rest of the configuration see ./config/application.properties -# Name (only) of SSL server certificate file in PKS format placed in ./keystore -SERVER_CERTIFICATE=<<>>> +# Name (only) of SSL server certificate file in PKS format placed in ./app-keys +SERVER_CERTIFICATE=ddrcore.p12 # Password for the server certificate file SERVER_CERT_PASSWORD=<<>> +# Token for authenticating to the vault +VAULT_PORT=<<8201>> + # Token for authenticating to the vault VAULT_TOKEN=<<>> # Hostname of the server running the vault (without protocol or port) -VAULT_HOSTNAME=<<>> +VAULT_HOSTNAME=vault + +# Endpoint URL for the S3 service +# This should be the full URL including protocol +# If using a custom S3-compatible service, ensure the URL is correct. +# Note: Do not include the bucket name in this URL. +S3_ENDPOINT_URL=<> # Specifies the version of the Key-Value secrets engine used in Vault. # 1 = KV Version 1 (no versioning of secrets) # 2 = KV Version 2 (versioned secrets support) -KV_BACKEND_VERSION=<<>> +KV_BACKEND_VERSION=1 # The mount path of the Key-Value secrets engine in Vault. KV_SECRET_PATH=<<>> +# APP port +# This is the port on which the application will run. +APP_PORT=443 + # API key for the service -API_KEY=<<>> +APP_API_KEY=<<>> # API secret for the service -API_SECRET=<<>> +APP_API_SECRET=<<>> # Project namespace (defaults to the current folder name if not set) -COMPOSE_PROJECT_NAME=s3point \ No newline at end of file +COMPOSE_PROJECT_NAME=ddr \ No newline at end of file diff --git a/deploy/app-keys/ddrcore.p12.toput b/deploy/app-keys/ddrcore.p12.toput new file mode 100644 index 0000000..68cc000 --- /dev/null +++ b/deploy/app-keys/ddrcore.p12.toput @@ -0,0 +1 @@ +Standard .p12 key file for SpringBoot app. Can be created from .pem files \ No newline at end of file diff --git a/deploy/config/application.properties b/deploy/config/application.properties index 3a9958c..6f3b7c2 100644 --- a/deploy/config/application.properties +++ b/deploy/config/application.properties @@ -7,28 +7,26 @@ management.endpoints.web.exposure.include=info # Logging logging.level.org.springframework=INFO # SSL -server.ssl.key-store=file:keystore/${SERVER_CERTIFICATE} +server.ssl.key-store=file:app-keys/${SERVER_CERTIFICATE} server.ssl.key-store-password=${SERVER_CERT_PASSWORD} server.ssl.key-store-type=PKCS12 server.ssl.enabled=true # Server -server.port=443 +server.port=${APP_PORT} server.http2.enabled=true server.shutdown=graceful -api.key=${API_KEY} -api.secret=${API_SECRET} +app.api-key=${APP_API_KEY} +app.api-secret=${APP_API_SECRET} # Multipart files spring.servlet.multipart.max-file-size=10MB spring.servlet.multipart.max-request-size=10MB # Profile - test profile is for development purpose with auth disable. -spring.profiles.active=production # stage, test +spring.profiles.active=production # CESNET S3 -s3.endpoint_url=<> -# name of the bucket where to upload registration images of instruments -s3.cdn_bucket_name=cxi-web +s3.endpoint-url=${S3_ENDPOINT_URL} # HashiCorp Vault -vault.port=8201 +vault.port=${VAULT_PORT} vault.token=${VAULT_TOKEN} vault.hostname=${VAULT_HOSTNAME} vault.kv.backend-version=${KV_BACKEND_VERSION} -vault.kv.secrets.path=${KV_SECRET_PATH} \ No newline at end of file +vault.kv.secrets-path=${KV_SECRET_PATH} \ No newline at end of file diff --git a/deploy/config/vault.hcl b/deploy/config/vault.hcl index b743209..3f65223 100644 --- a/deploy/config/vault.hcl +++ b/deploy/config/vault.hcl @@ -1,12 +1,12 @@ ui = true -api_addr = "https://omidb.cxi.tul.cz:8200" +api_addr = "https://0.0.0.0:8200" listener "tcp" { address = "0.0.0.0:8201" tls_disable = false - tls_cert_file = "/vault/config/ssl/s3core.crt", - tls_key_file = "/vault/config/ssl/s3corewp.key" + tls_cert_file = "/vault/config/ssl/ddrcore.crt", + tls_key_file = "/vault/config/ssl/ddrcore_decrypted.key" } storage "file" { diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 0b0abfd..7737051 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -8,25 +8,18 @@ services: depends_on: vault: condition: service_started - image: cxiomi/s3point-core:stage # x.x.x, stage + image: cxiomi/ddr-core:stage # x.x.x, stage ports: - - "443:443" + - ${APP_PORT}:${APP_PORT} volumes: - ./config/application.properties:/app/config/application.properties # directory to place application.properties - - ./keystore/:/app/keystore # directory to place all certs + - ./app-keys/:/app/app-keys # directory to place all certs + env_file: + - .env entrypoint: - java - - -Dspring.profiles.active=production # production, stage, test - - -DSERVER_CERTIFICATE=${SERVER_CERTIFICATE} - - -DSERVER_CERT_PASSWORD=${SERVER_CERT_PASSWORD} - - -DVAULT_TOKEN=${VAULT_TOKEN} - - -DVAULT_HOSTNAME=${VAULT_HOSTNAME} - - -DAPI_KEY=${API_KEY} - - -DAPI_SECRET=${API_SECRET} - - -DKV_BACKEND_VERSION=${KV_BACKEND_VERSION} - - -DKV_SECRET_PATH=${KV_SECRET_PATH} - -jar - - S3PointCore.jar + - DDRcore.jar healthcheck: test: "curl -I -k https://localhost:443" interval: 10s @@ -42,17 +35,17 @@ services: image: hashicorp/vault:latest hostname: ${VAULT_HOSTNAME} ports: - - "8200:8200" - - "8201:8201" - environment: - VAULT_TOKEN: ${VAULT_TOKEN} + - ${VAULT_PORT}:${VAULT_PORT} + - 8200:8200 + env_file: + - .env cap_add: - IPC_LOCK volumes: - data-volume:/vault/file - config-volume:/vault/config - ./config/vault.hcl:/vault/config/vault.hcl - - ./ssl/:/vault/config/ssl/ + - ./vault-keys/:/vault/config/ssl/ - ./config/init.sh:/vault/config/init.sh entrypoint: sh /vault/config/init.sh logging: *default-logging diff --git a/deploy/keystore/s3core.p12 b/deploy/keystore/s3core.p12 deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/ssl/s3core.crt b/deploy/ssl/s3core.crt deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/ssl/s3corewp.key b/deploy/ssl/s3corewp.key deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/vault-keys/ddrcore.crt.toput b/deploy/vault-keys/ddrcore.crt.toput new file mode 100644 index 0000000..46243ae --- /dev/null +++ b/deploy/vault-keys/ddrcore.crt.toput @@ -0,0 +1 @@ +Signed public key generated for DDRcore for use by Vault \ No newline at end of file diff --git a/deploy/vault-keys/ddrcore_decrypted.key.toput b/deploy/vault-keys/ddrcore_decrypted.key.toput new file mode 100644 index 0000000..bc6636b --- /dev/null +++ b/deploy/vault-keys/ddrcore_decrypted.key.toput @@ -0,0 +1 @@ +Private key generated for DDRcore for use by Vault. Decrypted (without password) \ No newline at end of file diff --git a/develop/.env.tofill b/develop/.env.tofill new file mode 100644 index 0000000..f4470fa --- /dev/null +++ b/develop/.env.tofill @@ -0,0 +1,34 @@ +# This file is for setting secrets and confidential information +# For the rest of the configuration see ./config/application.properties + +# APP port +APP_PORT=80 + +# Server address +# This is the address where the application will be accessible. +# It should include the protocol (http or https) and the port if not default. +SERVER_ADDRESS=http://localhost:${APP_PORT} + +# Token for authenticating to the vault +VAULT_PORT=<<>> + +# Token for authenticating to the vault +VAULT_TOKEN=root + +# Hostname of the server running the vault (without protocol or port) +VAULT_HOSTNAME=vault + +# Endpoint URL for the S3 service +# This should be the full URL including protocol +S3_ENDPOINT_URL=<> + +# Specifies the version of the Key-Value secrets engine used in Vault. +# 1 = KV Version 1 (no versioning of secrets) +# 2 = KV Version 2 (versioned secrets support) +KV_BACKEND_VERSION=1 + +# The mount path of the Key-Value secrets engine in Vault. +KV_SECRET_PATH=kv + +# Project namespace (defaults to the current folder name if not set) +COMPOSE_PROJECT_NAME=ddr-test \ No newline at end of file diff --git a/develop/config/application.properties b/develop/config/application.properties index 7d53fec..fd62d01 100644 --- a/develop/config/application.properties +++ b/develop/config/application.properties @@ -3,19 +3,19 @@ # Environment variables are set by Docker at start # Logging -logging.level.cz.tul.cxi.S3PointCore=DEBUG +logging.level.cz.tul.cxi.DDRcore=DEBUG logging.level.org.springframework=DEBUG # Server -server.port=80 +server.port=${APP_PORT} # Multipart files spring.servlet.multipart.max-file-size=10MB spring.servlet.multipart.max-request-size=10MB # Profile - test profile is for development purpose with auth disable. spring.profiles.active=test # CESNET S3 -s3.endpoint_url=https://s3.cl4.du.cesnet.cz +s3.endpoint-url=${S3_ENDPOINT_URL} # HashiCorp Vault -vault.port=8201 +vault.port=${VAULT_PORT} vault.token=${VAULT_TOKEN} vault.kv.backend-version=${KV_BACKEND_VERSION} -vault.kv.secrets.path=${KV_SECRET_PATH} \ No newline at end of file +vault.kv.secrets-path=${KV_SECRET_PATH} \ No newline at end of file diff --git a/develop/docker-compose.yaml b/develop/docker-compose.yaml index bd3a6a7..bbfd0e4 100644 --- a/develop/docker-compose.yaml +++ b/develop/docker-compose.yaml @@ -6,29 +6,28 @@ services: dockerfile: ./build-release/Dockerfile context: .. args: - - TARGET_FILE=S3PointCore.jar + - TARGET_FILE=DDRcore.jar ports: - - "81:80" - - "5005:5005" + - ${APP_PORT}:${APP_PORT} + - 5005:5005 volumes: - ./config/application.properties:/app/config/application.properties # directory to place application.properties - - ./keystore/:/app/keystore # directory to place all certs + - ./app-keys/:/app/app-keys # directory to place all certs environment: JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + env_file: + - .env entrypoint: - java - - -DVAULT_TOKEN=${VAULT_TOKEN} - - -DKV_BACKEND_VERSION=${KV_BACKEND_VERSION} - - -DKV_SECRET_PATH=${KV_SECRET_PATH} - -jar - - S3PointCore.jar + - DDRcore.jar pull_policy: always vault: image: hashicorp/vault:latest ports: - - "8200:8200" - - "8201:8201" + - 8200:8200 + - ${VAULT_PORT}:${VAULT_PORT} environment: VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_TOKEN} cap_add: diff --git a/develop/fill.env b/develop/fill.env deleted file mode 100644 index 71cecd9..0000000 --- a/develop/fill.env +++ /dev/null @@ -1,12 +0,0 @@ -# This file is for setting secrets and confidential information -# For the rest of the configuration see ./config/application.properties - -# Rename this file to .env and fill in the values for your project - -# Token for authenticating to the vault -VAULT_TOKEN=root -KV_BACKEND_VERSION=2 -KV_SECRET_PATH=secret - -# Project namespace (defaults to the current folder name if not set) -COMPOSE_PROJECT_NAME=s3point-test2 \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index bcbda54..72e3152 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1 @@ -rootProject.name = 'S3PointCore' +rootProject.name = 'DDRcore' diff --git a/src/main/java/cz/tul/cxi/DDRcore/DDRcoreApplication.java b/src/main/java/cz/tul/cxi/DDRcore/DDRcoreApplication.java new file mode 100644 index 0000000..56067a9 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/DDRcoreApplication.java @@ -0,0 +1,12 @@ +package cz.tul.cxi.DDRcore; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DDRcoreApplication { + + public static void main(String[] args) { + SpringApplication.run(DDRcoreApplication.class, args); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/component/ApiKeyAuthFilter.java b/src/main/java/cz/tul/cxi/DDRcore/component/ApiKeyAuthFilter.java new file mode 100644 index 0000000..138d737 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/component/ApiKeyAuthFilter.java @@ -0,0 +1,43 @@ +package cz.tul.cxi.DDRcore.component; + +import java.io.IOException; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +@Component +@Profile(value = {"stage", "production"}) +public class ApiKeyAuthFilter extends OncePerRequestFilter { + @Value("${app.api-key}") + private String apiKey; + + @Value("${app.api-secret}") + private String apiSecret; + + @Override + protected void doFilterInternal( + HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + + // Get the API key and secret from request headers + String requestApiKey = request.getHeader("X-API-KEY"); + String requestApiSecret = request.getHeader("X-API-SECRET"); + // Validate the key and secret + if (apiKey.equals(requestApiKey) && apiSecret.equals(requestApiSecret)) { + // Continue processing the request + filterChain.doFilter(request, response); + } else { + // Reject the request and send an unauthorized error + response.setStatus(HttpStatus.UNAUTHORIZED.value()); + response.getWriter().write("Unauthorized"); + } + } +} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/component/JSONComponent.java b/src/main/java/cz/tul/cxi/DDRcore/component/JSONComponent.java similarity index 95% rename from src/main/java/cz/tul/cxi/S3PointCore/component/JSONComponent.java rename to src/main/java/cz/tul/cxi/DDRcore/component/JSONComponent.java index 8169c63..515b69b 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/component/JSONComponent.java +++ b/src/main/java/cz/tul/cxi/DDRcore/component/JSONComponent.java @@ -1,8 +1,8 @@ -package cz.tul.cxi.S3PointCore.component; +package cz.tul.cxi.DDRcore.component; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import cz.tul.cxi.S3PointCore.model.BucketPolicy; +import cz.tul.cxi.DDRcore.model.BucketPolicy; import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/src/main/java/cz/tul/cxi/DDRcore/component/S3ClientProvider.java b/src/main/java/cz/tul/cxi/DDRcore/component/S3ClientProvider.java new file mode 100644 index 0000000..8b6a289 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/component/S3ClientProvider.java @@ -0,0 +1,94 @@ +package cz.tul.cxi.DDRcore.component; + +import java.net.URI; +import java.util.Objects; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.springframework.vault.core.VaultKeyValueOperations; +import org.springframework.vault.core.VaultKeyValueOperationsSupport; +import org.springframework.vault.core.VaultTemplate; +import org.springframework.vault.support.VaultResponse; +import org.springframework.web.context.annotation.RequestScope; + +import cz.tul.cxi.DDRcore.exception.CredentialsNotFoundException; +import cz.tul.cxi.DDRcore.exception.MissingAdminEinfraIdException; +import cz.tul.cxi.DDRcore.exception.S3ClientInitializationException; +import jakarta.servlet.http.HttpServletRequest; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; + +@Component +public class S3ClientProvider { + + private final VaultTemplate vaultTemplate; + + @Value("${s3.endpoint-url}") + private String endpointURL; + + @Value("${vault.kv.backend-version:1}") // default to KV_1 + private int kvBackendVersion; + + @Value("${vault.kv.secrets-path:kv}") + private String secretsPath; + + public S3ClientProvider(VaultTemplate vaultTemplate) { + this.vaultTemplate = vaultTemplate; + } + + @Bean + @RequestScope + public S3Client cesnetS3Client(HttpServletRequest request) { + String einfraId = request.getHeader("Admin-Einfra-Id"); + + if (einfraId == null || einfraId.isBlank()) { + throw new MissingAdminEinfraIdException("Missing 'Admin-Einfra-Id' header in request"); + } + + try { + AwsCredentialsProvider provider = credentialsProvider(einfraId); + + return S3Client.builder() + .credentialsProvider(provider) + .endpointOverride(URI.create(endpointURL)) + .region(Region.AWS_GLOBAL) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) + .build(); + } catch (CredentialsNotFoundException ex) { + throw ex; // delegate, controller advice will catch this exception + } catch (Exception ex) { + throw new S3ClientInitializationException( + "Failed to initialize S3 client for e-infraId: %s. Message: %s." + .formatted(einfraId, ex.getMessage()), + ex); + } + } + + private AwsCredentialsProvider credentialsProvider(String einfraId) { + VaultKeyValueOperations keyValueOperations = + vaultTemplate.opsForKeyValue( + secretsPath, + (kvBackendVersion == 1) + ? VaultKeyValueOperationsSupport.KeyValueBackend.KV_1 + : VaultKeyValueOperationsSupport.KeyValueBackend.KV_2); + VaultResponse response = keyValueOperations.get(einfraId); + + if (response == null + || response.getData() == null + || !response.getData().containsKey("access_key") + || !response.getData().containsKey("secret_key")) { + throw new CredentialsNotFoundException(einfraId); + } + + String accessKey = (String) Objects.requireNonNull(response.getData()).get("access_key"); + String secretKey = (String) response.getData().get("secret_key"); + return StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)); + } +} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/component/S3Component.java b/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java similarity index 90% rename from src/main/java/cz/tul/cxi/S3PointCore/component/S3Component.java rename to src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java index 5c1197c..b12f643 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/component/S3Component.java +++ b/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java @@ -1,9 +1,15 @@ -package cz.tul.cxi.S3PointCore.component; +package cz.tul.cxi.DDRcore.component; -import cz.tul.cxi.S3PointCore.model.BucketPolicy; -import cz.tul.cxi.S3PointCore.model.S3Bucket; -import cz.tul.cxi.S3PointCore.model.S3Object; -import cz.tul.cxi.S3PointCore.model.Statement; +import cz.tul.cxi.DDRcore.model.BucketPolicy; +import cz.tul.cxi.DDRcore.model.S3Bucket; +import cz.tul.cxi.DDRcore.model.S3Object; +import cz.tul.cxi.DDRcore.model.Statement; +import java.nio.file.Path; +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import software.amazon.awssdk.core.ResponseInputStream; @@ -17,13 +23,6 @@ import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequ import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; -import java.nio.file.Path; -import java.time.Duration; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - @Component @Slf4j public class S3Component { @@ -65,7 +64,7 @@ public class S3Component { s3Client.createBucket(bucketRequest); } - public void putDefaultBucketCors(String bucketName) throws S3Exception{ + public void putDefaultBucketCors(String bucketName) throws S3Exception { PutBucketCorsRequest corsRequest = PutBucketCorsRequest.builder() .bucket(bucketName) @@ -86,7 +85,8 @@ public class S3Component { } public String createDownloadUrl( - String bucketName, String prefix, String name, int signatureDurationMinutes) throws SdkClientException { + String bucketName, String prefix, String name, int signatureDurationMinutes) + throws SdkClientException { try (S3Presigner presigner = S3Presigner.builder() .region(s3Client.serviceClientConfiguration().region()) @@ -152,14 +152,15 @@ public class S3Component { ListObjectsV2Request.builder().bucket(bucketName).prefix(fullPrefix).delimiter("/").build(); try { ListObjectsV2Response res = s3Client.listObjectsV2(listObjects); - String datasetZipFileName = name.isBlank() ? String.format("%s%s.zip", prefix, prefix.replace("/", "")) : ""; + String datasetZipFileName = + name.isBlank() ? String.format("%s%s.zip", prefix, prefix.replace("/", "")) : ""; log.debug("List current level objects response: {}", res); for (software.amazon.awssdk.services.s3.model.S3Object object : res.contents()) { - if (object.key().equals(fullPrefix) || !datasetZipFileName.isEmpty() && datasetZipFileName.equals(object.key())) - continue; + if (object.key().equals(fullPrefix) + || !datasetZipFileName.isEmpty() && datasetZipFileName.equals(object.key())) continue; - files.add(new S3Object(bucketName, createName(object.key(), fullPrefix), fullPrefix)); + files.add(new S3Object(bucketName, createName(object.key(), fullPrefix), fullPrefix)); } for (CommonPrefix object : res.commonPrefixes()) { files.add(new S3Object(bucketName, createName(object.prefix(), fullPrefix), fullPrefix)); @@ -211,13 +212,12 @@ public class S3Component { private String createName(String key, String fullPrefix) { key = key.replaceFirst(fullPrefix, ""); - if(key.contains("/")){ + if (key.contains("/")) { String[] splitName = key.split("/"); String name = splitName[splitName.length - 1]; name = key.endsWith("/") ? name + "/" : name; return name; - } - else return key; + } else return key; } public List listBuckets() { @@ -260,24 +260,19 @@ public class S3Component { return objects.stream().toList(); } - public ResponseInputStream getObject(String bucketName, String key) throws S3Exception{ - GetObjectRequest getRequest = GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(); + public ResponseInputStream getObject(String bucketName, String key) + throws S3Exception { + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucketName).key(key).build(); return s3Client.getObject(getRequest); } public void deleteObject(String bucketName, String key) throws S3Exception { - s3Client.deleteObject(DeleteObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build()); - + s3Client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(key).build()); } public void putZipObject(String bucketName, String key, Path sourcePath) throws S3Exception { - PutObjectRequest putRequest = PutObjectRequest.builder() + PutObjectRequest putRequest = + PutObjectRequest.builder() .bucket(bucketName) .key(key) .contentType("application/zip") diff --git a/src/main/java/cz/tul/cxi/S3PointCore/config/SecurityConfig.java b/src/main/java/cz/tul/cxi/DDRcore/config/SecurityConfig.java similarity index 59% rename from src/main/java/cz/tul/cxi/S3PointCore/config/SecurityConfig.java rename to src/main/java/cz/tul/cxi/DDRcore/config/SecurityConfig.java index 3f6e9d8..4ffa6af 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/config/SecurityConfig.java +++ b/src/main/java/cz/tul/cxi/DDRcore/config/SecurityConfig.java @@ -1,4 +1,4 @@ -package cz.tul.cxi.S3PointCore.config; +package cz.tul.cxi.DDRcore.config; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -10,23 +10,21 @@ import org.springframework.security.config.annotation.web.configurers.AbstractHt import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; -import cz.tul.cxi.S3PointCore.component.ApiKeyAuthFilter; +import cz.tul.cxi.DDRcore.component.ApiKeyAuthFilter; @Configuration @EnableWebSecurity @Profile(value = {"stage", "production"}) public class SecurityConfig { - @Autowired - private ApiKeyAuthFilter apiKeyAuthFilter; + @Autowired private ApiKeyAuthFilter apiKeyAuthFilter; - @Bean - SecurityFilterChain app(HttpSecurity http) throws Exception { - http.csrf(AbstractHttpConfigurer::disable); - http.addFilterBefore(apiKeyAuthFilter, UsernamePasswordAuthenticationFilter.class); - http.authorizeHttpRequests((authorize) -> authorize - .anyRequest().permitAll()); + @Bean + SecurityFilterChain app(HttpSecurity http) throws Exception { + http.csrf(AbstractHttpConfigurer::disable); + http.addFilterBefore(apiKeyAuthFilter, UsernamePasswordAuthenticationFilter.class); + http.authorizeHttpRequests((authorize) -> authorize.anyRequest().permitAll()); - return http.build(); - } + return http.build(); + } } diff --git a/src/main/java/cz/tul/cxi/S3PointCore/config/SecurityTestConfig.java b/src/main/java/cz/tul/cxi/DDRcore/config/SecurityTestConfig.java similarity index 65% rename from src/main/java/cz/tul/cxi/S3PointCore/config/SecurityTestConfig.java rename to src/main/java/cz/tul/cxi/DDRcore/config/SecurityTestConfig.java index de3e105..864a70b 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/config/SecurityTestConfig.java +++ b/src/main/java/cz/tul/cxi/DDRcore/config/SecurityTestConfig.java @@ -1,4 +1,4 @@ -package cz.tul.cxi.S3PointCore.config; +package cz.tul.cxi.DDRcore.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -12,11 +12,10 @@ import org.springframework.security.web.SecurityFilterChain; @EnableWebSecurity @Profile("test") public class SecurityTestConfig { - @Bean - SecurityFilterChain app(HttpSecurity http) throws Exception { - http.authorizeHttpRequests((authorize) -> authorize - .anyRequest().permitAll()); - http.csrf(AbstractHttpConfigurer::disable); - return http.build(); - } + @Bean + SecurityFilterChain app(HttpSecurity http) throws Exception { + http.authorizeHttpRequests((authorize) -> authorize.anyRequest().permitAll()); + http.csrf(AbstractHttpConfigurer::disable); + return http.build(); + } } diff --git a/src/main/java/cz/tul/cxi/DDRcore/config/VaultConfig.java b/src/main/java/cz/tul/cxi/DDRcore/config/VaultConfig.java new file mode 100644 index 0000000..f02f7bb --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/config/VaultConfig.java @@ -0,0 +1,34 @@ +package cz.tul.cxi.DDRcore.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.vault.authentication.TokenAuthentication; +import org.springframework.vault.client.VaultEndpoint; +import org.springframework.vault.core.VaultTemplate; + +// https://docs.spring.io/spring-vault/reference/vault/vault-repositories.html + +@Configuration +@Profile(value = {"stage", "production"}) +class VaultConfig { + + @Value("${vault.port}") + private int vaultPort; + + @Value("${vault.token}") + private String vaultToken; + + @Value("${vault.hostname}") + private String vaultHostname; + + @Bean + public VaultTemplate vaultTemplate() { + VaultEndpoint endpoint = new VaultEndpoint(); + endpoint.setScheme("https"); + endpoint.setHost(vaultHostname); + endpoint.setPort(vaultPort); + return new VaultTemplate(endpoint, new TokenAuthentication(vaultToken)); + } +} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/config/VaultTestConfig.java b/src/main/java/cz/tul/cxi/DDRcore/config/VaultTestConfig.java similarity index 54% rename from src/main/java/cz/tul/cxi/S3PointCore/config/VaultTestConfig.java rename to src/main/java/cz/tul/cxi/DDRcore/config/VaultTestConfig.java index 95cec71..8453309 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/config/VaultTestConfig.java +++ b/src/main/java/cz/tul/cxi/DDRcore/config/VaultTestConfig.java @@ -1,4 +1,4 @@ -package cz.tul.cxi.S3PointCore.config; +package cz.tul.cxi.DDRcore.config; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -14,19 +14,18 @@ import org.springframework.vault.core.VaultTemplate; @Profile(value = {"test"}) class VaultTestConfig { - @Value("${vault.port}") - private int vaultPort; + @Value("${vault.port}") + private int vaultPort; - @Value("${vault.token}") - private String vaultToken; + @Value("${vault.token}") + private String vaultToken; - @Bean - public VaultTemplate vaultTemplate() { - VaultEndpoint endpoint = new VaultEndpoint(); - endpoint.setScheme("http"); - endpoint.setHost("vault"); - endpoint.setPort(vaultPort); - return new VaultTemplate(endpoint, new TokenAuthentication(vaultToken)); - } + @Bean + public VaultTemplate vaultTemplate() { + VaultEndpoint endpoint = new VaultEndpoint(); + endpoint.setScheme("http"); + endpoint.setHost("vault"); + endpoint.setPort(vaultPort); + return new VaultTemplate(endpoint, new TokenAuthentication(vaultToken)); + } } - diff --git a/src/main/java/cz/tul/cxi/S3PointCore/controller/RestController.java b/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java similarity index 91% rename from src/main/java/cz/tul/cxi/S3PointCore/controller/RestController.java rename to src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java index 58160dd..4f9d200 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/controller/RestController.java +++ b/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java @@ -1,11 +1,11 @@ -package cz.tul.cxi.S3PointCore.controller; +package cz.tul.cxi.DDRcore.controller; import com.fasterxml.jackson.core.JsonProcessingException; -import cz.tul.cxi.S3PointCore.model.Credentials; -import cz.tul.cxi.S3PointCore.model.S3Bucket; -import cz.tul.cxi.S3PointCore.model.S3Object; -import cz.tul.cxi.S3PointCore.service.CredentialsService; -import cz.tul.cxi.S3PointCore.service.S3Service; +import cz.tul.cxi.DDRcore.model.Credentials; +import cz.tul.cxi.DDRcore.model.S3Bucket; +import cz.tul.cxi.DDRcore.model.S3Object; +import cz.tul.cxi.DDRcore.service.CredentialsService; +import cz.tul.cxi.DDRcore.service.S3Service; import jakarta.validation.Valid; import java.io.IOException; import org.springframework.http.HttpStatus; @@ -83,14 +83,13 @@ public class RestController { @PutMapping("/buckets/updatePolicy") public ResponseEntity updateBucketPolicy( - @RequestParam String bucketName, - @RequestBody String policyJson) { + @RequestParam String bucketName, @RequestBody String policyJson) { try { s3Service.updateBucketPolicy(bucketName, policyJson); return ResponseEntity.ok("Bucket policy updated successfully."); } catch (Exception e) { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Error updating bucket policy: " + e.getMessage()); + .body("Error updating bucket policy: " + e.getMessage()); } } @@ -242,7 +241,8 @@ public class RestController { return handleAccessDenied(einfraId, bucketName); } return ResponseEntity.ok( - s3Service.getObjectDownloadPublicLink(bucketName, prefix, name, signatureDurationMinutes)); + s3Service.getObjectDownloadPublicLink( + bucketName, prefix, name, signatureDurationMinutes)); } catch (JsonProcessingException e) { return handleIOException( e, @@ -257,23 +257,24 @@ public class RestController { @GetMapping(value = "/objects/downloadLog", consumes = "application/json") public ResponseEntity getInstrumentLog( - @RequestParam(value = "bucketName") @NonNull String bucketName, - @RequestParam(value = "name") @NonNull String name, - @RequestParam(value = "einfraId") @NonNull String einfraId, - @RequestParam(name = "signatureDurationMinutes", required = false, defaultValue = "60") + @RequestParam(value = "bucketName") @NonNull String bucketName, + @RequestParam(value = "name") @NonNull String name, + @RequestParam(value = "einfraId") @NonNull String einfraId, + @RequestParam(name = "signatureDurationMinutes", required = false, defaultValue = "60") int signatureDurationMinutes) { try { verifyVaultCredentials(einfraId); return ResponseEntity.ok( - s3Service.getObjectDownloadPublicLink(bucketName, "logs/", name, signatureDurationMinutes)); + s3Service.getObjectDownloadPublicLink( + bucketName, "logs/", name, signatureDurationMinutes)); } catch (JsonProcessingException e) { return handleIOException( - e, - "Failed to parse JSON for object with bucketName: %s, name: %s, prefix: logs/." - .formatted(bucketName, name)); + e, + "Failed to parse JSON for object with bucketName: %s, name: %s, prefix: logs/." + .formatted(bucketName, name)); } catch (SdkClientException e) { return handleSdkException(e, bucketName, "logs/", name); - } catch (S3Exception ex) { + } catch (S3Exception ex) { return handleS3Exception(ex, bucketName); } } @@ -310,7 +311,7 @@ public class RestController { @DeleteMapping("/objects/delete") public ResponseEntity deleteObject( - @RequestParam String bucketName, @RequestParam String key) { + @RequestParam String bucketName, @RequestParam String key) { s3Service.deleteObject(bucketName, key); return ResponseEntity.ok("Object deleted successfully."); } diff --git a/src/main/java/cz/tul/cxi/DDRcore/exception/CredentialsNotFoundException.java b/src/main/java/cz/tul/cxi/DDRcore/exception/CredentialsNotFoundException.java new file mode 100644 index 0000000..df63b08 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/exception/CredentialsNotFoundException.java @@ -0,0 +1,7 @@ +package cz.tul.cxi.DDRcore.exception; + +public class CredentialsNotFoundException extends RuntimeException { + public CredentialsNotFoundException(String einfraId) { + super("Vault credentials not found for einfraId: " + einfraId); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/exception/DDRcoreExceptionHandler.java b/src/main/java/cz/tul/cxi/DDRcore/exception/DDRcoreExceptionHandler.java new file mode 100644 index 0000000..dbcab34 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/exception/DDRcoreExceptionHandler.java @@ -0,0 +1,28 @@ +package cz.tul.cxi.DDRcore.exception; + +import java.util.Map; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +@RestControllerAdvice +public class DDRcoreExceptionHandler { + + @ExceptionHandler(MissingAdminEinfraIdException.class) + public ResponseEntity handleMissingHeader(MissingAdminEinfraIdException ex) { + return ResponseEntity.badRequest() + .body(Map.of("error", "Missing Tenant-Id header", "details", ex.getMessage())); + } + + @ExceptionHandler(CredentialsNotFoundException.class) + public ResponseEntity handleCredentialsNotFound(CredentialsNotFoundException ex) { + return ResponseEntity.badRequest() + .body(Map.of("error", "Credentials not found", "details", ex.getMessage())); + } + + @ExceptionHandler(S3ClientInitializationException.class) + public ResponseEntity handleS3ClientInit(S3ClientInitializationException ex) { + return ResponseEntity.badRequest() + .body(Map.of("error", "S3 client initialization failed", "details", ex.getMessage())); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/exception/MissingAdminEinfraIdException.java b/src/main/java/cz/tul/cxi/DDRcore/exception/MissingAdminEinfraIdException.java new file mode 100644 index 0000000..f13367a --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/exception/MissingAdminEinfraIdException.java @@ -0,0 +1,7 @@ +package cz.tul.cxi.DDRcore.exception; + +public class MissingAdminEinfraIdException extends RuntimeException { + public MissingAdminEinfraIdException(String message) { + super(message); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/exception/S3ClientInitializationException.java b/src/main/java/cz/tul/cxi/DDRcore/exception/S3ClientInitializationException.java new file mode 100644 index 0000000..25e2e5c --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/exception/S3ClientInitializationException.java @@ -0,0 +1,7 @@ +package cz.tul.cxi.DDRcore.exception; + +public class S3ClientInitializationException extends RuntimeException { + public S3ClientInitializationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/model/BucketPolicy.java b/src/main/java/cz/tul/cxi/DDRcore/model/BucketPolicy.java new file mode 100644 index 0000000..bb65e54 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/model/BucketPolicy.java @@ -0,0 +1,11 @@ +package cz.tul.cxi.DDRcore.model; + +import java.util.List; + +public record BucketPolicy( + String Id, String Version, List Statement) { + + public BucketPolicy(List Statement) { + this("PolicyId", "2012-10-17", Statement); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/model/Credentials.java b/src/main/java/cz/tul/cxi/DDRcore/model/Credentials.java new file mode 100644 index 0000000..4476359 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/model/Credentials.java @@ -0,0 +1,22 @@ +package cz.tul.cxi.DDRcore.model; + +import jakarta.validation.constraints.NotBlank; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class Credentials { + @NotBlank(message = "einfraId must not be blank.") + private String einfraId; + + @NotBlank(message = "s3AccessKey must not be blank.") + private String s3AccessKey; + + @NotBlank(message = "s3SecretKey must not be blank.") + private String s3SecretKey; +} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/model/S3Bucket.java b/src/main/java/cz/tul/cxi/DDRcore/model/S3Bucket.java similarity index 66% rename from src/main/java/cz/tul/cxi/S3PointCore/model/S3Bucket.java rename to src/main/java/cz/tul/cxi/DDRcore/model/S3Bucket.java index 21f3792..d0d9298 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/model/S3Bucket.java +++ b/src/main/java/cz/tul/cxi/DDRcore/model/S3Bucket.java @@ -1,4 +1,4 @@ -package cz.tul.cxi.S3PointCore.model; +package cz.tul.cxi.DDRcore.model; import jakarta.validation.constraints.NotBlank; import lombok.AllArgsConstructor; @@ -11,6 +11,6 @@ import lombok.Setter; @AllArgsConstructor @NoArgsConstructor public class S3Bucket { - @NotBlank(message = "(bucket) name must not be blank.") - private String name; + @NotBlank(message = "(bucket) name must not be blank.") + private String name; } diff --git a/src/main/java/cz/tul/cxi/DDRcore/model/S3Object.java b/src/main/java/cz/tul/cxi/DDRcore/model/S3Object.java new file mode 100644 index 0000000..2dadf9e --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/model/S3Object.java @@ -0,0 +1,45 @@ +package cz.tul.cxi.DDRcore.model; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Setter +@Getter +@AllArgsConstructor +@NoArgsConstructor +public class S3Object { + @NotBlank(message = "bucketName must not be blank.") + private String bucketName; + + @NotBlank(message = "(file) name must not be blank.") + private String name; // current patch of path aka display name + + @NotNull(message = "(file) prefix must not be null.") + private String prefix; // almost full path relative to bucketName + + @Override + public int hashCode() { + return bucketName.hashCode() + name.hashCode() + prefix.hashCode(); + } + + @Override + public boolean equals(Object object) { + if (object == null) { + return false; + } + if (object == this) { + return true; + } + if (object.getClass() != this.getClass()) { + return false; + } + S3Object s3Object = (S3Object) object; + return s3Object.getBucketName().equals(this.getBucketName()) + && s3Object.getName().equals(this.getName()) + && s3Object.getPrefix().equals(this.getPrefix()); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/model/Statement.java b/src/main/java/cz/tul/cxi/DDRcore/model/Statement.java new file mode 100644 index 0000000..3a96151 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/model/Statement.java @@ -0,0 +1,12 @@ +package cz.tul.cxi.DDRcore.model; + +import java.util.List; +import java.util.Map; + +public record Statement( + String Sid, + List Action, + String Effect, + Map> Principal, + List Resource, + Map>> Condition) {} diff --git a/src/main/java/cz/tul/cxi/DDRcore/service/CredentialsService.java b/src/main/java/cz/tul/cxi/DDRcore/service/CredentialsService.java new file mode 100644 index 0000000..5bc01fe --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/service/CredentialsService.java @@ -0,0 +1,83 @@ +package cz.tul.cxi.DDRcore.service; + +import cz.tul.cxi.DDRcore.exception.CredentialsNotFoundException; +import cz.tul.cxi.DDRcore.model.Credentials; +import jakarta.annotation.PostConstruct; +import java.util.Map; +import java.util.Objects; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.vault.core.VaultKeyValueOperations; +import org.springframework.vault.core.VaultKeyValueOperationsSupport; +import org.springframework.vault.core.VaultTemplate; +import org.springframework.vault.support.VaultResponse; + +@Service +public class CredentialsService { + + private final VaultTemplate vaultTemplate; + + private VaultKeyValueOperations keyValueOperations; + + @Value("${vault.kv.backend-version:1}") // default to KV_1 + private int kvBackendVersion; + + @Value("${vault.kv.secrets-path:kv}") + private String secretsPath; + + public CredentialsService(VaultTemplate vaultTemplate) { + this.vaultTemplate = vaultTemplate; + } + + @PostConstruct + public void init() { + this.keyValueOperations = + vaultTemplate.opsForKeyValue( + secretsPath, + (kvBackendVersion == 1) + ? VaultKeyValueOperationsSupport.KeyValueBackend.KV_1 + : VaultKeyValueOperationsSupport.KeyValueBackend.KV_2); + } + + public void putCredentials(Credentials credentials) { + keyValueOperations.put( + credentials.getEinfraId(), + Map.of( + "access_key", + credentials.getS3AccessKey(), + "secret_key", + credentials.getS3SecretKey())); + } + + public void patchCredentials(Credentials credentials) { + // KV 1 does not support patching, so we overwrite the existing entry + putCredentials(credentials); + } + + public Credentials getCredentials(String eInfraId) { + VaultResponse response = keyValueOperations.get(eInfraId); + + if (response == null + || response.getData() == null + || !response.getData().containsKey("access_key") + || !response.getData().containsKey("secret_key")) { + throw new CredentialsNotFoundException(eInfraId); + } + + return new Credentials( + eInfraId, + (String) Objects.requireNonNull(response.getData()).get("access_key"), + (String) response.getData().get("secret_key")); + } + + public void verifyVaultCredentials(String eInfraId) { + VaultResponse response = keyValueOperations.get(eInfraId); + + if (response == null + || response.getData() == null + || !response.getData().containsKey("access_key") + || !response.getData().containsKey("secret_key")) { + throw new CredentialsNotFoundException(eInfraId); + } + } +} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/service/S3Service.java b/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java similarity index 74% rename from src/main/java/cz/tul/cxi/S3PointCore/service/S3Service.java rename to src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java index cbc57c4..56fba1a 100644 --- a/src/main/java/cz/tul/cxi/S3PointCore/service/S3Service.java +++ b/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java @@ -1,18 +1,27 @@ -package cz.tul.cxi.S3PointCore.service; +package cz.tul.cxi.DDRcore.service; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import cz.tul.cxi.S3PointCore.component.JSONComponent; -import cz.tul.cxi.S3PointCore.component.S3Component; -import cz.tul.cxi.S3PointCore.model.*; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import lombok.extern.slf4j.Slf4j; + import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import cz.tul.cxi.DDRcore.component.JSONComponent; +import cz.tul.cxi.DDRcore.component.S3Component; +import cz.tul.cxi.DDRcore.model.BucketPolicy; +import cz.tul.cxi.DDRcore.model.S3Bucket; +import cz.tul.cxi.DDRcore.model.S3Object; +import cz.tul.cxi.DDRcore.model.Statement; +import lombok.extern.slf4j.Slf4j; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkException; @@ -35,11 +44,14 @@ public class S3Service { this.mapper = new ObjectMapper(); } - public void createBucket(S3Bucket s3Bucket, String tenantId, String einfraId, String adminEInfraId) throws S3Exception, IOException { + public void createBucket( + S3Bucket s3Bucket, String tenantId, String einfraId, String adminEInfraId) + throws S3Exception, IOException { s3Component.createBucket(s3Bucket.getName()); s3Component.putDefaultBucketCors(s3Bucket.getName()); s3Component.updateBucketPolicy( - s3Bucket.getName(), constructDefaultPolicy(s3Bucket.getName(), tenantId, einfraId, adminEInfraId)); + s3Bucket.getName(), + constructDefaultPolicy(s3Bucket.getName(), tenantId, einfraId, adminEInfraId)); } public String listDatasets(String bucketName, String tenantId, String einfraId) @@ -52,7 +64,8 @@ public class S3Service { return jsonComponent.createSimpleFrontendJSON(datasets); } - public void createDataset(S3Object s3Object, String tenantId, String einfraId, String adminEInfraId) + public void createDataset( + S3Object s3Object, String tenantId, String einfraId, String adminEInfraId) throws S3Exception, IOException { s3Component.createEmpty(s3Object.getBucketName(), s3Object.getPrefix(), s3Object.getName()); initDefaultDatasetShareStatement(s3Object, tenantId, adminEInfraId); @@ -70,24 +83,18 @@ public class S3Service { s3Component.createEmpty(s3Object.getBucketName(), s3Object.getPrefix(), s3Object.getName()); } - public String getObjectDownloadPublicLink(String bucketName, String prefix, String name, int signatureDurationMinutes) + public String getObjectDownloadPublicLink( + String bucketName, String prefix, String name, int signatureDurationMinutes) throws JsonProcessingException, SdkException { - String url = - s3Component.createDownloadUrl( - bucketName, - prefix, - name, - signatureDurationMinutes); + String url = s3Component.createDownloadUrl(bucketName, prefix, name, signatureDurationMinutes); return mapper.writeValueAsString(Map.of("url", url)); } public String getDownloadUrls(String bucketName, String prefix, String name) throws SdkClientException, JsonProcessingException { - Set files = - s3Component.listAllObjects( - bucketName, prefix, name); + Set files = s3Component.listAllObjects(bucketName, prefix, name); - if(name.isBlank()){ + if (name.isBlank()) { String zipFileName = String.format("%s.zip", prefix.replace("/", "")); files.removeIf(it -> zipFileName.equals(it.getName())); } @@ -103,8 +110,8 @@ public class S3Service { it.getBucketName(), it.getPrefix(), it.getName(), 60), "prefix", it.getPrefix(), - "fileName", - it.getName())) + "fileName", + it.getName())) .toList(); return mapper.writeValueAsString(Map.of("fileUrls", fileUrls)); @@ -162,29 +169,27 @@ public class S3Service { s3Component.updateBucketPolicy(s3dataset.getBucketName(), policyString); } - public void initDefaultDatasetShareStatement(S3Object s3dataset, String tenantId, String adminEInfraId) - throws S3Exception, IOException { + public void initDefaultDatasetShareStatement( + S3Object s3dataset, String tenantId, String adminEInfraId) throws S3Exception, IOException { BucketPolicy bucketPolicy = - jsonComponent.convertJsonStringToBucketPolicy( - s3Component.getCurrentBucketPolicy(s3dataset.getBucketName())); + jsonComponent.convertJsonStringToBucketPolicy( + s3Component.getCurrentBucketPolicy(s3dataset.getBucketName())); String awsPrincipal = String.format("arn:aws:iam::%s:user/%s", tenantId, adminEInfraId); - List statementTemplates = - jsonComponent.readJsonFileToBucketPolicy("policies/share_statements.json").Statement(); - for (Statement statementTemplate : statementTemplates) { - if (statementTemplate.Sid().contains("ReadWrite")) { - Statement statement = - updateDatasetStatements( - statementTemplate, s3dataset.getName(), s3dataset.getBucketName(), awsPrincipal); - bucketPolicy.Statement().add(statement); - } - else{ - Statement statement = - updateDatasetStatements( - statementTemplate, s3dataset.getName(), s3dataset.getBucketName(), ""); - bucketPolicy.Statement().add(statement); - } - + List statementTemplates = + jsonComponent.readJsonFileToBucketPolicy("policies/share_statements.json").Statement(); + for (Statement statementTemplate : statementTemplates) { + if (statementTemplate.Sid().contains("ReadWrite")) { + Statement statement = + updateDatasetStatements( + statementTemplate, s3dataset.getName(), s3dataset.getBucketName(), awsPrincipal); + bucketPolicy.Statement().add(statement); + } else { + Statement statement = + updateDatasetStatements( + statementTemplate, s3dataset.getName(), s3dataset.getBucketName(), ""); + bucketPolicy.Statement().add(statement); + } } String policyString = jsonComponent.convertBucketPolicyToJsonString(bucketPolicy); @@ -251,26 +256,29 @@ public class S3Service { return jsonComponent.createSimpleFrontendJSON(allowedBuckets); } - public boolean isDatasetAccessDenied(String tenantId, String einfraId, String bucketName, String prefix) throws S3Exception, JsonProcessingException { + public boolean isDatasetAccessDenied( + String tenantId, String einfraId, String bucketName, String prefix) + throws S3Exception, JsonProcessingException { String bucketPolicyString = s3Component.getCurrentBucketPolicy(bucketName); BucketPolicy bucketPolicy = jsonComponent.convertJsonStringToBucketPolicy(bucketPolicyString); String awsPrincipal = String.format("arn:aws:iam::%s:user/%s", tenantId, einfraId); String[] prefixes = prefix.split("/"); - if(prefixes.length == 0){ + if (prefixes.length == 0) { return false; } for (Statement statement : bucketPolicy.Statement()) { if (statement.Sid().contains(prefixes[0]) - && statement.Principal().get("AWS").contains(awsPrincipal)) { + && statement.Principal().get("AWS").contains(awsPrincipal)) { return false; } } return true; } - private String constructDefaultPolicy(String bucketName, String tenantId, String einfraId, String adminEIfraId) throws IOException { + private String constructDefaultPolicy( + String bucketName, String tenantId, String einfraId, String adminEIfraId) throws IOException { BucketPolicy bucketPolicyTemplate = jsonComponent.readJsonFileToBucketPolicy("policies/default_bucket_policy.json"); BucketPolicy bucketPolicy = new BucketPolicy(new ArrayList<>()); @@ -278,14 +286,16 @@ public class S3Service { String awsPrincipal = String.format("arn:aws:iam::%s:user/%s", tenantId, einfraId); for (Statement statementTemplate : bucketPolicyTemplate.Statement()) { - Statement statement; - if (statementTemplate.Sid().contains("ListBucket")){ - statement = updateBucketStatements(statementTemplate, bucketName, List.of(adminAwsPrincipal, awsPrincipal)); - } - else{ - statement = updateBucketStatements(statementTemplate, bucketName, List.of(adminAwsPrincipal)); - } - bucketPolicy.Statement().add(statement); + Statement statement; + if (statementTemplate.Sid().contains("ListBucket")) { + statement = + updateBucketStatements( + statementTemplate, bucketName, List.of(adminAwsPrincipal, awsPrincipal)); + } else { + statement = + updateBucketStatements(statementTemplate, bucketName, List.of(adminAwsPrincipal)); + } + bucketPolicy.Statement().add(statement); } String policyString = jsonComponent.convertBucketPolicyToJsonString(bucketPolicy); @@ -316,17 +326,15 @@ public class S3Service { (it, it2) -> it2.forEach( (it3, it4) -> - it4.replaceAll(it5 -> it5.replace("DATASET_FOLDER", datasetName)))); + it4.replaceAll(it5 -> it5.replace("DATASET_FOLDER", datasetName)))); return new Statement( - statementTemplate.Sid().replace("DATASET_FOLDER", datasetName), + statementTemplate.Sid().replace("DATASET_FOLDER", datasetName), statementTemplate.Action(), statementTemplate.Effect(), Map.of("AWS", List.of(awsPrincipal)), statementTemplate.Resource().stream() - .map( - it -> - it.replace("BUCKET_NAME", bucketName).replace("DATASET_FOLDER", datasetName)) + .map(it -> it.replace("BUCKET_NAME", bucketName).replace("DATASET_FOLDER", datasetName)) .toList(), statementTemplate.Condition()); } @@ -340,18 +348,19 @@ public class S3Service { } } - public String getDatasetDownloadLink(String bucketName, String prefix, int signatureDurationMinutes) throws IOException, S3Exception { + public String getDatasetDownloadLink( + String bucketName, String prefix, int signatureDurationMinutes) + throws IOException, S3Exception { String zipFileName = String.format("%s%s.zip", prefix, prefix.replace("/", "")); s3Component.deleteObject(bucketName, zipFileName); - Set files = - s3Component.listAllObjects( - bucketName, prefix, ""); + Set files = s3Component.listAllObjects(bucketName, prefix, ""); files.removeIf(it -> it.getName().endsWith("/")); File zipFile = File.createTempFile(prefix, ".zip"); try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { for (S3Object object : files) { - try (ResponseInputStream s3Stream = s3Component.getObject(bucketName, object.getPrefix() + object.getName())) { + try (ResponseInputStream s3Stream = + s3Component.getObject(bucketName, object.getPrefix() + object.getName())) { String key = object.getPrefix() + object.getName(); String relativePath = key.substring(prefix.length()).replaceFirst("^/", ""); @@ -364,20 +373,22 @@ public class S3Service { } s3Component.putZipObject(bucketName, zipFileName, zipFile.toPath()); - String url = s3Component.createDownloadUrl(bucketName, "", zipFileName, signatureDurationMinutes); + String url = + s3Component.createDownloadUrl(bucketName, "", zipFileName, signatureDurationMinutes); List> fileUrls = - List.of(Map.of( - "url", - url, - "prefix", - prefix, - "fileName", - "%s.zip".formatted(prefix.replace("/", "")))); + List.of( + Map.of( + "url", + url, + "prefix", + prefix, + "fileName", + "%s.zip".formatted(prefix.replace("/", "")))); return mapper.writeValueAsString(Map.of("fileUrls", fileUrls)); } - public void deleteObject(String bucketName, String key){ + public void deleteObject(String bucketName, String key) { s3Component.deleteObject(bucketName, key); } diff --git a/src/main/java/cz/tul/cxi/S3PointCore/S3PointCoreApplication.java b/src/main/java/cz/tul/cxi/S3PointCore/S3PointCoreApplication.java deleted file mode 100644 index 8aff92b..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/S3PointCoreApplication.java +++ /dev/null @@ -1,13 +0,0 @@ -package cz.tul.cxi.S3PointCore; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class S3PointCoreApplication { - - public static void main(String[] args) { - SpringApplication.run(S3PointCoreApplication.class, args); - } -} - diff --git a/src/main/java/cz/tul/cxi/S3PointCore/component/ApiKeyAuthFilter.java b/src/main/java/cz/tul/cxi/S3PointCore/component/ApiKeyAuthFilter.java deleted file mode 100644 index 0ffd787..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/component/ApiKeyAuthFilter.java +++ /dev/null @@ -1,39 +0,0 @@ -package cz.tul.cxi.S3PointCore.component; - -import jakarta.servlet.FilterChain; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Profile; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Component; -import org.springframework.web.filter.OncePerRequestFilter; - -import java.io.IOException; - -@Component -@Profile(value = {"stage", "production"}) -public class ApiKeyAuthFilter extends OncePerRequestFilter { - @Value("${api.key}") - private String apiKey; - @Value("${api.secret}") - private String apiSecret; - @Override - protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) - throws ServletException, IOException { - - // Get the API key and secret from request headers - String requestApiKey = request.getHeader("X-API-KEY"); - String requestApiSecret = request.getHeader("X-API-SECRET"); - // Validate the key and secret - if (apiKey.equals(requestApiKey) && apiSecret.equals(requestApiSecret)) { - // Continue processing the request - filterChain.doFilter(request, response); - } else { - // Reject the request and send an unauthorized error - response.setStatus(HttpStatus.UNAUTHORIZED.value()); - response.getWriter().write("Unauthorized"); - } - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/component/S3ClientProvider.java b/src/main/java/cz/tul/cxi/S3PointCore/component/S3ClientProvider.java deleted file mode 100644 index fca903c..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/component/S3ClientProvider.java +++ /dev/null @@ -1,91 +0,0 @@ -package cz.tul.cxi.S3PointCore.component; - -import java.net.URI; -import java.util.Objects; - -import cz.tul.cxi.S3PointCore.exception.CredentialsNotFoundException; -import cz.tul.cxi.S3PointCore.exception.MissingAdminEinfraIdException; -import cz.tul.cxi.S3PointCore.exception.S3ClientInitializationException; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.springframework.vault.core.VaultKeyValueOperations; -import org.springframework.vault.core.VaultKeyValueOperationsSupport; -import org.springframework.vault.core.VaultTemplate; -import org.springframework.vault.support.VaultResponse; -import org.springframework.web.context.annotation.RequestScope; - -import jakarta.servlet.http.HttpServletRequest; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3Configuration; - -@Component -public class S3ClientProvider { - - private final VaultTemplate vaultTemplate; - - @Value("${s3.endpoint_url}") - private String endpointURL; - - @Value("${vault.kv.backend-version:1}") // default to KV_1 - private int kvBackendVersion; - - @Value("${vault.kv.secrets.path:kv}") - private String secretsPath; - - public S3ClientProvider(VaultTemplate vaultTemplate) { - this.vaultTemplate = vaultTemplate; - } - - @Bean - @RequestScope - public S3Client cesnetS3Client(HttpServletRequest request) { - String einfraId = request.getHeader("Admin-Einfra-Id"); - - if (einfraId == null || einfraId.isBlank()) { - throw new MissingAdminEinfraIdException("Missing 'Admin-Einfra-Id' header in request"); - } - - try { - AwsCredentialsProvider provider = credentialsProvider(einfraId); - - return S3Client.builder() - .credentialsProvider(provider) - .endpointOverride(URI.create(endpointURL)) - .region(Region.AWS_GLOBAL) - .serviceConfiguration(S3Configuration.builder() - .pathStyleAccessEnabled(true) - .build()) - .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED) - .build(); - } catch (CredentialsNotFoundException ex) { - throw ex; // delegate, controller advice will catch this exception - } catch (Exception ex) { - throw new S3ClientInitializationException("Failed to initialize S3 client for e-infraId: %s. Message: %s.".formatted(einfraId, ex.getMessage()), ex); - } - } - - private AwsCredentialsProvider credentialsProvider(String einfraId) { - VaultKeyValueOperations keyValueOperations = vaultTemplate.opsForKeyValue(secretsPath, - (kvBackendVersion == 1) - ? VaultKeyValueOperationsSupport.KeyValueBackend.KV_1 - : VaultKeyValueOperationsSupport.KeyValueBackend.KV_2); - VaultResponse response = keyValueOperations.get(einfraId); - - if (response == null || response.getData() == null || - !response.getData().containsKey("access_key") || - !response.getData().containsKey("secret_key")) { - throw new CredentialsNotFoundException(einfraId); - } - - String accessKey = (String) Objects.requireNonNull(response.getData()).get("access_key"); - String secretKey = (String) response.getData().get("secret_key"); - return StaticCredentialsProvider - .create(AwsBasicCredentials.create(accessKey, secretKey)); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/config/VaultConfig.java b/src/main/java/cz/tul/cxi/S3PointCore/config/VaultConfig.java deleted file mode 100644 index 985ee1f..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/config/VaultConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -package cz.tul.cxi.S3PointCore.config; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; -import org.springframework.vault.authentication.ClientAuthentication; -import org.springframework.vault.authentication.TokenAuthentication; -import org.springframework.vault.client.VaultEndpoint; -import org.springframework.vault.core.VaultTemplate; -import org.springframework.vault.repository.configuration.EnableVaultRepositories; - -// https://docs.spring.io/spring-vault/reference/vault/vault-repositories.html - -@Configuration -@Profile(value = {"stage", "production"}) -class VaultConfig { - - @Value("${vault.port}") - private int vaultPort; - - @Value("${vault.token}") - private String vaultToken; - - @Value("${vault.hostname}") - private String vaultHostname; - - @Bean - public VaultTemplate vaultTemplate() { - VaultEndpoint endpoint = new VaultEndpoint(); - endpoint.setScheme("https"); - endpoint.setHost(vaultHostname); - endpoint.setPort(vaultPort); - return new VaultTemplate(endpoint, new TokenAuthentication(vaultToken)); - } -} - diff --git a/src/main/java/cz/tul/cxi/S3PointCore/exception/CredentialsNotFoundException.java b/src/main/java/cz/tul/cxi/S3PointCore/exception/CredentialsNotFoundException.java deleted file mode 100644 index 24eac92..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/exception/CredentialsNotFoundException.java +++ /dev/null @@ -1,7 +0,0 @@ -package cz.tul.cxi.S3PointCore.exception; - -public class CredentialsNotFoundException extends RuntimeException { - public CredentialsNotFoundException(String einfraId) { - super("Vault credentials not found for einfraId: " + einfraId); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/exception/MissingAdminEinfraIdException.java b/src/main/java/cz/tul/cxi/S3PointCore/exception/MissingAdminEinfraIdException.java deleted file mode 100644 index 62c90d1..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/exception/MissingAdminEinfraIdException.java +++ /dev/null @@ -1,7 +0,0 @@ -package cz.tul.cxi.S3PointCore.exception; - -public class MissingAdminEinfraIdException extends RuntimeException { - public MissingAdminEinfraIdException(String message) { - super(message); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/exception/S3ClientInitializationException.java b/src/main/java/cz/tul/cxi/S3PointCore/exception/S3ClientInitializationException.java deleted file mode 100644 index cc11ddb..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/exception/S3ClientInitializationException.java +++ /dev/null @@ -1,7 +0,0 @@ -package cz.tul.cxi.S3PointCore.exception; - -public class S3ClientInitializationException extends RuntimeException { - public S3ClientInitializationException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/exception/S3PointExceptionHandler.java b/src/main/java/cz/tul/cxi/S3PointCore/exception/S3PointExceptionHandler.java deleted file mode 100644 index 49894f3..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/exception/S3PointExceptionHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -package cz.tul.cxi.S3PointCore.exception; - -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -import java.util.Map; - -@RestControllerAdvice -public class S3PointExceptionHandler { - - @ExceptionHandler(MissingAdminEinfraIdException.class) - public ResponseEntity handleMissingHeader(MissingAdminEinfraIdException ex) { - return ResponseEntity.badRequest().body(Map.of( - "error", "Missing Tenant-Id header", - "details", ex.getMessage() - )); - } - - @ExceptionHandler(CredentialsNotFoundException.class) - public ResponseEntity handleCredentialsNotFound(CredentialsNotFoundException ex) { - return ResponseEntity.badRequest().body(Map.of( - "error", "Credentials not found", - "details", ex.getMessage() - )); - } - - @ExceptionHandler(S3ClientInitializationException.class) - public ResponseEntity handleS3ClientInit(S3ClientInitializationException ex) { - return ResponseEntity.badRequest().body(Map.of( - "error", "S3 client initialization failed", - "details", ex.getMessage() - )); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/model/BucketPolicy.java b/src/main/java/cz/tul/cxi/S3PointCore/model/BucketPolicy.java deleted file mode 100644 index 00cc8b6..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/model/BucketPolicy.java +++ /dev/null @@ -1,12 +0,0 @@ -package cz.tul.cxi.S3PointCore.model; - -import java.util.List; - -public record BucketPolicy(String Id, - String Version, - List Statement) { - - public BucketPolicy(List Statement) { - this("PolicyId", "2012-10-17", Statement); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/model/Credentials.java b/src/main/java/cz/tul/cxi/S3PointCore/model/Credentials.java deleted file mode 100644 index 055cde0..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/model/Credentials.java +++ /dev/null @@ -1,20 +0,0 @@ -package cz.tul.cxi.S3PointCore.model; - - -import jakarta.validation.constraints.NotBlank; -import lombok.*; - -@Getter -@Setter -@NoArgsConstructor -@AllArgsConstructor -public class Credentials { - @NotBlank(message = "einfraId must not be blank.") - private String einfraId; - - @NotBlank(message = "s3AccessKey must not be blank.") - private String s3AccessKey; - - @NotBlank(message = "s3SecretKey must not be blank.") - private String s3SecretKey; -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/model/S3Object.java b/src/main/java/cz/tul/cxi/S3PointCore/model/S3Object.java deleted file mode 100644 index 3a71607..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/model/S3Object.java +++ /dev/null @@ -1,45 +0,0 @@ -package cz.tul.cxi.S3PointCore.model; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Setter -@Getter -@AllArgsConstructor -@NoArgsConstructor -public class S3Object { - @NotBlank(message = "bucketName must not be blank.") - private String bucketName; - - @NotBlank(message = "(file) name must not be blank.") - private String name; // current patch of path aka display name - - @NotNull(message = "(file) prefix must not be null.") - private String prefix; // almost full path relative to bucketName - - @Override - public int hashCode() { - return bucketName.hashCode() + name.hashCode() + prefix.hashCode(); - } - - @Override - public boolean equals(Object object) { - if (object == null) { - return false; - } - if (object == this) { - return true; - } - if (object.getClass() != this.getClass()) { - return false; - } - S3Object s3Object = (S3Object) object; - return s3Object.getBucketName().equals(this.getBucketName()) && - s3Object.getName().equals(this.getName()) && - s3Object.getPrefix().equals(this.getPrefix()); - } -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/model/Statement.java b/src/main/java/cz/tul/cxi/S3PointCore/model/Statement.java deleted file mode 100644 index ff2a797..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/model/Statement.java +++ /dev/null @@ -1,12 +0,0 @@ -package cz.tul.cxi.S3PointCore.model; - -import java.util.List; -import java.util.Map; - -public record Statement(String Sid, - List Action, - String Effect, - Map> Principal, - List Resource, - Map>> Condition) { -} diff --git a/src/main/java/cz/tul/cxi/S3PointCore/service/CredentialsService.java b/src/main/java/cz/tul/cxi/S3PointCore/service/CredentialsService.java deleted file mode 100644 index c4f3d7f..0000000 --- a/src/main/java/cz/tul/cxi/S3PointCore/service/CredentialsService.java +++ /dev/null @@ -1,75 +0,0 @@ -package cz.tul.cxi.S3PointCore.service; - -import cz.tul.cxi.S3PointCore.exception.CredentialsNotFoundException; -import cz.tul.cxi.S3PointCore.model.Credentials; -import jakarta.annotation.PostConstruct; -import java.util.Map; -import java.util.Objects; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.vault.core.VaultKeyValueOperations; -import org.springframework.vault.core.VaultKeyValueOperationsSupport; -import org.springframework.vault.core.VaultTemplate; -import org.springframework.vault.support.VaultResponse; - -@Service -public class CredentialsService { - - private final VaultTemplate vaultTemplate; - - private VaultKeyValueOperations keyValueOperations; - - @Value("${vault.kv.backend-version:1}") // default to KV_1 - private int kvBackendVersion; - - @Value("${vault.kv.secrets.path:kv}") - private String secretsPath; - - public CredentialsService(VaultTemplate vaultTemplate) { - this.vaultTemplate = vaultTemplate; - } - - @PostConstruct - public void init() { - this.keyValueOperations = vaultTemplate.opsForKeyValue(secretsPath, - (kvBackendVersion == 1) - ? VaultKeyValueOperationsSupport.KeyValueBackend.KV_1 - : VaultKeyValueOperationsSupport.KeyValueBackend.KV_2); - } - - public void putCredentials(Credentials credentials) { - keyValueOperations.put( - credentials.getEinfraId(), - Map.of("access_key", credentials.getS3AccessKey(), "secret_key", credentials.getS3SecretKey())); - } - - public void patchCredentials(Credentials credentials) { - // KV 1 does not support patching, so we overwrite the existing entry - putCredentials(credentials); - } - - public Credentials getCredentials(String eInfraId) { - VaultResponse response = keyValueOperations.get(eInfraId); - - if (response == null || response.getData() == null || - !response.getData().containsKey("access_key") || - !response.getData().containsKey("secret_key")) { - throw new CredentialsNotFoundException(eInfraId); - } - - return new Credentials( - eInfraId, - (String) Objects.requireNonNull(response.getData()).get("access_key"), - (String) response.getData().get("secret_key")); - } - - public void verifyVaultCredentials(String eInfraId) { - VaultResponse response = keyValueOperations.get(eInfraId); - - if (response == null || response.getData() == null || - !response.getData().containsKey("access_key") || - !response.getData().containsKey("secret_key")) { - throw new CredentialsNotFoundException(eInfraId); - } - } -} diff --git a/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/src/main/resources/META-INF/additional-spring-configuration-metadata.json new file mode 100644 index 0000000..8d9372f --- /dev/null +++ b/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -0,0 +1,48 @@ +{ + "properties": [ + { + "name": "app.api-key", + "type": "java.lang.String", + "description": "API key for accessing the application" + }, + { + "name": "app.api-secret", + "type": "java.lang.String", + "description": "API secret for accessing the application" + }, + { + "name": "s3.endpoint-url", + "type": "java.lang.String", + "description": "Endpoint URL for the S3 service", + "defaultValue": "https://s3.cl4.du.cesnet.cz" + }, + { + "name": "vault.port", + "type": "java.lang.String", + "description": "Port for the Vault service", + "defaultValue": "8201" + }, + { + "name": "vault.token", + "type": "java.lang.String", + "description": "Token for accessing the Vault service" + }, + { + "name": "vault.hostname", + "type": "java.lang.String", + "description": "Hostname for the Vault service" + }, + { + "name": "vault.kv.backend-version", + "type": "java.lang.String", + "description": "Version of the Vault KV backend", + "defaultValue": "1" + }, + { + "name": "vault.kv.secrets-path", + "type": "java.lang.String", + "description": "Path to the secrets in the Vault KV store", + "defaultValue": "kv" + } + ] +} \ No newline at end of file diff --git a/src/test/java/cz/tul/cxi/DDRcore/DDRcoreApplicationTests.java b/src/test/java/cz/tul/cxi/DDRcore/DDRcoreApplicationTests.java new file mode 100644 index 0000000..a57c5d6 --- /dev/null +++ b/src/test/java/cz/tul/cxi/DDRcore/DDRcoreApplicationTests.java @@ -0,0 +1,13 @@ +package cz.tul.cxi.DDRcore; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = DDRcoreApplication.class) +class DDRcoreApplicationTests { + + @Test + void contextLoads() {} +} diff --git a/src/test/java/cz/tul/cxi/DDRcore/TestDDRcoreApplication.java b/src/test/java/cz/tul/cxi/DDRcore/TestDDRcoreApplication.java new file mode 100644 index 0000000..dd0d7cb --- /dev/null +++ b/src/test/java/cz/tul/cxi/DDRcore/TestDDRcoreApplication.java @@ -0,0 +1,12 @@ +package cz.tul.cxi.DDRcore; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.test.context.TestConfiguration; + +@TestConfiguration(proxyBeanMethods = false) +public class TestDDRcoreApplication { + + public static void main(String[] args) { + SpringApplication.from(DDRcoreApplication::main).with(TestDDRcoreApplication.class).run(args); + } +} diff --git a/src/test/java/cz/tul/cxi/S3PointCore/S3PointCoreApplicationTests.java b/src/test/java/cz/tul/cxi/S3PointCore/S3PointCoreApplicationTests.java deleted file mode 100644 index 8e51742..0000000 --- a/src/test/java/cz/tul/cxi/S3PointCore/S3PointCoreApplicationTests.java +++ /dev/null @@ -1,18 +0,0 @@ -package cz.tul.cxi.S3PointCore; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -@ExtendWith(SpringExtension.class) -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = S3PointCoreApplication.class) -class S3PointCoreApplicationTests { - - @Test - void contextLoads() { - } - -} diff --git a/src/test/java/cz/tul/cxi/S3PointCore/TestS3PointCoreApplication.java b/src/test/java/cz/tul/cxi/S3PointCore/TestS3PointCoreApplication.java deleted file mode 100644 index 4039af0..0000000 --- a/src/test/java/cz/tul/cxi/S3PointCore/TestS3PointCoreApplication.java +++ /dev/null @@ -1,13 +0,0 @@ -package cz.tul.cxi.S3PointCore; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.test.context.TestConfiguration; - -@TestConfiguration(proxyBeanMethods = false) -public class TestS3PointCoreApplication { - - public static void main(String[] args) { - SpringApplication.from(S3PointCoreApplication::main).with(TestS3PointCoreApplication.class).run(args); - } - -} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 0d65c53..684b604 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -4,7 +4,7 @@ server.http2.enabled=true # Profile spring.profiles.active=test # CESNET S3 -s3.endpoint_url=https://s3.cl4.du.cesnet.cz +s3.endpoint-url=https://s3.cl4.du.cesnet.cz s3.vo_name=vo_name s3.admin_name=e_infra_id s3.access_key="access" -- GitLab From 2643e885bbd7304d9ec4dae5454ce468ee387b1c Mon Sep 17 00:00:00 2001 From: Venceslav Chumchal Date: Tue, 29 Jul 2025 12:30:59 +0200 Subject: [PATCH 2/5] Revert logo back MSMT logo --- README.md | 159 ++---------------------------------------------------- 1 file changed, 5 insertions(+), 154 deletions(-) diff --git a/README.md b/README.md index f506ba0..044f803 100644 --- a/README.md +++ b/README.md @@ -1,159 +1,10 @@ -# DDRcore - a core services for DDRplatform +

EOSC CZ Logo

-## Description - -Web service developed within the CESNET project S3Point implemented at CXI TUL. It complements a DDR ecosystem. The backbone of the web application is the Java framework Spring Boot. As a repository of data needed for the application is used HashiCorp Vault (key-value engine). The application is built using Gradle, containerized using Docker, and deployed using -Docker Compose. - -[Detailed information about tool in separate wiki](../../../wikis/home). - -## Deployment Guide - -This guide provides detailed instructions for deploying the DDRcore web service with Docker Compose and configuring HashiCorp Vault for secure data management. - ---- - -### 1. Configuration - -1. **Download and Arrange Deployment Files**: - Begin by downloading the `deploy` directory to your target server. This directory should contain: - - **docker-compose.yaml**: Defines Docker Compose configuration. - - **application.properties**: Configures application-specific settings for DDRcore. - - **.env**: Contains environment variables for Docker and the application, including tokens and credentials. - - **app-keys and vault-keys directories**: Place public/private keys and certificates here. - - **app-keys**: Stores keys in Java format. - - **vault-keys**: Stores certificates and keys in PEM format. - -2. **S3 and Vault Access Configuration**: - In `.env`, locate the placeholder values (marked as `<<>>`) and replace them with: - - **Vault access**: For securely storing and retrieving sensitive application data. - - **S3 access**: Required for backend access to Amazon S3. - - Other requested deployment information. - -3. **Set Certificates and Keys**: - Ensure the paths to certificates and keys are correctly set in the `app-keys` and `vault-keys` directories. - ---- - -### 2. Initial Deployment and First-time Vault Setup - -1. **Start Containers**: - Run the following command to start the application containers: - ```bash - docker compose up --wait - - -This will initialize the DDRcore and Vault containers. The `--wait` flag ensures Docker waits until all containers are fully up and running. - -### 2. Vault Configuration (First-time Setup): -1. Open the Vault GUI by navigating to `https://your-domain.com:8201`. - -2. Initiliaze Vault: -Vault is initially sealed for security. To intialize and unseal: -- Generate unseal keys. -- Download generated keys and root token as JSON. -- Enter the unseal keys generated during initialization (multiple keys may be required). -- Store these keys securely for future use, as they are needed if Vault is restarted. - -3. Obtain a Root Token: -After unsealing, Vault issues a root token for initial access. Store this token securely, as it’s needed for configuring Vault and must be provided to the application. - -4. Set Up Key-Value (KV) Database: -- Create a Key-Value (KV) secrets engine called `kv` (see Vault KV documentation). -- Populate the database with required credentials and secrets: - - **e-infra ID** (record name): The ID associated with your infrastructure. - - **access_key** and **secret_key** (key-value pairs): Corresponding S3 credentials. - -## 3. Restart Application with Vault Token - -1. Insert Vault Token in .env: -Copy the root token obtained in step 2 into the `.env` file under `VAULT_TOKEN`. - -2. Restart the Web Service Container: -- Terminate the existing DDRcore container and restart: - ```bash - docker kill ddr-core-1 - docker compose up --wait - -This restart initializes the application with access to Vault, allowing DDRcore to read secrets from the KV database. - -Maintenance Note: Unsealing Vault After Updates -Vault Auto-Unsealing: -The current setup does not support Vault auto-unsealing. After any Vault update or restart, it will return to a sealed state. Repeat the unsealing process in the GUI or command line as outlined in Step 2. Always keep unseal keys accessible to avoid prolonged downtime due to a locked Vault after updates. - - - -## Development - -To build and deploy as a Docker image locally, you first need to fill *develop/config/application.properties* and -*develop/.env* files. - -Then for local develoment instance run -`` -./local-deploy.sh -`` - -When running DDRcore locally with vault note that it's in dev mode and dev mode does not support persistent storage – all data (secrets, policies, tokens, etc.) are stored in memory only and will be lost when the container restarts. - -**After each start:** -Access the container using: -`` -docker exec -it ddr-test2-vault-1 sh -`` - -Set the `VAULT_ADDR` environment variable manually: -`` -export VAULT_ADDR='http://127.0.0.1:8200' -`` - -Add secrets: -`` -vault kv put secret/<<>> access_key=<<>> secret_key=<<>> -`` *** -To build a Java jar you need to run the following: -`` -./build-release/build.sh test -`` - -Then in the *develop* directory, run: -`` -docker compose up --wait -`` - -For a production build, you need to run the following: -`` -./build-release/build.sh {target version} -`` -Note: The target versions used by our developer team are -**production** and **stage**. - -The container registry is used to release -on [Docker Hub](https://hub.docker.com/r/cxiomi/ddr-core) with the hub name **cxiomi**, where the -images are uploaded for subsequent deployment. The application is developed so that the actual deployment can be done -with only a change of configuration files in the *deploy* directory. In the case of modifying the code and building an -image for your institution, you need to use your container repository. - -The release can be triggered by: -`` -./build-release/release.sh {hub} {target version} -`` - -The application can be built and released at the same time using the following: -`` -./build-release/build-and-release.sh {hub} {target version} -`` - - - -## Authors - -Jan Kočí - lead, architect -Věnceslav Chumchal - developer, architect -David Vobruba - developer -Jakub Zach - consultant +## Description +This project output was developed with financial contributions from the EOSC CZ initiative through the project National Repository Platform for Research Data (CZ.02.01.01/00/23_014/0008787), funded by the Programme Johannes Amos Comenius (P JAC) of the Ministry of Education, Youth and Sports of the Czech Republic (MEYS). -## License +*** -Apache-2.0 +

EU and MŠMT Logos

-- GitLab From 719550bb7d281a8c86b170b874bde9fb75a5ac78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anna=20Volkov=C3=A1?= Date: Fri, 1 Aug 2025 13:42:02 +0000 Subject: [PATCH 3/5] New upload file endpoint --- .../cxi/DDRcore/component/S3Component.java | 12 ++++++++++++ .../DDRcore/controller/RestController.java | 19 +++++++++++++++++++ .../cz/tul/cxi/DDRcore/service/S3Service.java | 5 +++++ 3 files changed, 36 insertions(+) diff --git a/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java b/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java index b12f643..48f7e4f 100644 --- a/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java +++ b/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java @@ -4,6 +4,8 @@ import cz.tul.cxi.DDRcore.model.BucketPolicy; import cz.tul.cxi.DDRcore.model.S3Bucket; import cz.tul.cxi.DDRcore.model.S3Object; import cz.tul.cxi.DDRcore.model.Statement; + +import java.io.IOException; import java.nio.file.Path; import java.time.Duration; import java.util.HashSet; @@ -12,6 +14,7 @@ import java.util.Map; import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.sync.RequestBody; @@ -279,4 +282,13 @@ public class S3Component { .build(); s3Client.putObject(putRequest, sourcePath); } + + public void putFile(String bucketName, String key, MultipartFile image) throws IOException, S3Exception { + PutObjectRequest objectRequest = + PutObjectRequest.builder().bucket(bucketName).key(key).build(); + + s3Client.putObject( + objectRequest, + RequestBody.fromInputStream(image.getInputStream(), image.getSize())); + } } diff --git a/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java b/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java index 4f9d200..dfd04d4 100644 --- a/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java +++ b/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java @@ -13,6 +13,7 @@ import org.springframework.http.HttpStatusCode; import org.springframework.http.ResponseEntity; import org.springframework.lang.NonNull; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.model.S3Exception; @@ -320,6 +321,24 @@ public class RestController { credentialsService.verifyVaultCredentials(eInfraId); } + @PostMapping("/file/upload") + public ResponseEntity uploadFile( + @RequestParam @NonNull String bucketName, + @RequestParam @NonNull String name, + @RequestParam @NonNull String einfraId, + @RequestBody MultipartFile image) { + try { + verifyVaultCredentials(einfraId); + s3Service.putFile(bucketName, name, image); + return ResponseEntity.ok("File uploaded."); + } catch (S3Exception ex) { + return handleS3Exception(ex, bucketName); + } catch (IOException e) { + return handleIOException( + e, "IOException for bucketName: %s, name: %s.".formatted(bucketName, name)); + } + } + private ResponseEntity handleS3Exception(S3Exception ex, String bucketName) { if ("NoSuchBucketPolicy".equals(ex.awsErrorDetails().errorCode())) { return ResponseEntity.status(HttpStatus.NOT_FOUND) diff --git a/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java b/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java index 56fba1a..c375d6a 100644 --- a/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java +++ b/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java @@ -22,6 +22,7 @@ import cz.tul.cxi.DDRcore.model.S3Bucket; import cz.tul.cxi.DDRcore.model.S3Object; import cz.tul.cxi.DDRcore.model.Statement; import lombok.extern.slf4j.Slf4j; +import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkException; @@ -395,4 +396,8 @@ public class S3Service { public void updateBucketPolicy(String bucketName, String policyJson) { s3Component.updateBucketPolicy(bucketName, policyJson); } + + public void putFile(String bucketName, String name, MultipartFile image) throws IOException, S3Exception { + s3Component.putFile(bucketName, name, image); + } } -- GitLab From c36ef0a3ccdfff9a09c1ff92ef137e3f04866c34 Mon Sep 17 00:00:00 2001 From: Venceslav Chumchal Date: Mon, 4 Aug 2025 11:36:40 +0200 Subject: [PATCH 4/5] Update gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 95877e9..93b7680 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,5 @@ out/ ### VS Code ### .vscode/ /develop/.env -vault-cluster-vault-2024-03-29T19_23_51.689Z.json -vault-cluster-vault-2025-05-20T07_47_01.633Z.json +vault-kadi-prod.json +vault-omidb-stage.json -- GitLab From f55a025b80152f0703c7223724708407a8139493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anna=20Volkov=C3=A1?= Date: Tue, 26 Aug 2025 08:09:11 +0000 Subject: [PATCH 5/5] Upload file endpoint --- .../cxi/DDRcore/component/S3Component.java | 18 ++++++- .../DDRcore/controller/RestController.java | 51 +++++++++++++++---- .../exception/FileExistsException.java | 7 +++ .../cz/tul/cxi/DDRcore/service/S3Service.java | 13 ++++- 4 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 src/main/java/cz/tul/cxi/DDRcore/exception/FileExistsException.java diff --git a/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java b/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java index 48f7e4f..75ea65e 100644 --- a/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java +++ b/src/main/java/cz/tul/cxi/DDRcore/component/S3Component.java @@ -283,12 +283,26 @@ public class S3Component { s3Client.putObject(putRequest, sourcePath); } - public void putFile(String bucketName, String key, MultipartFile image) throws IOException, S3Exception { + public void putFile(String bucketName, String key, MultipartFile file) throws IOException, S3Exception { PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(bucketName).key(key).build(); s3Client.putObject( objectRequest, - RequestBody.fromInputStream(image.getInputStream(), image.getSize())); + RequestBody.fromInputStream(file.getInputStream(), file.getSize())); + } + + public boolean fileExists(String bucketName, String key) throws S3Exception { + try { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + s3Client.headObject(headObjectRequest); + return true; + } catch (NoSuchKeyException e) { + return false; + } } } diff --git a/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java b/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java index dfd04d4..ca9136f 100644 --- a/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java +++ b/src/main/java/cz/tul/cxi/DDRcore/controller/RestController.java @@ -1,6 +1,7 @@ package cz.tul.cxi.DDRcore.controller; import com.fasterxml.jackson.core.JsonProcessingException; +import cz.tul.cxi.DDRcore.exception.FileExistsException; import cz.tul.cxi.DDRcore.model.Credentials; import cz.tul.cxi.DDRcore.model.S3Bucket; import cz.tul.cxi.DDRcore.model.S3Object; @@ -312,24 +313,25 @@ public class RestController { @DeleteMapping("/objects/delete") public ResponseEntity deleteObject( - @RequestParam String bucketName, @RequestParam String key) { - s3Service.deleteObject(bucketName, key); - return ResponseEntity.ok("Object deleted successfully."); - } - - private void verifyVaultCredentials(String eInfraId) { - credentialsService.verifyVaultCredentials(eInfraId); + @RequestParam String bucketName, @RequestParam String name, @RequestParam @NonNull String einfraId) { + try { + verifyVaultCredentials(einfraId); + s3Service.deleteObject(bucketName, name); + return ResponseEntity.ok("Object deleted successfully."); + } catch (S3Exception ex) { + return handleS3Exception(ex, bucketName); + } } - @PostMapping("/file/upload") + @PostMapping("/file/upsert") public ResponseEntity uploadFile( @RequestParam @NonNull String bucketName, @RequestParam @NonNull String name, @RequestParam @NonNull String einfraId, - @RequestBody MultipartFile image) { + @RequestBody MultipartFile file) { try { verifyVaultCredentials(einfraId); - s3Service.putFile(bucketName, name, image); + s3Service.putFile(bucketName, name, file); return ResponseEntity.ok("File uploaded."); } catch (S3Exception ex) { return handleS3Exception(ex, bucketName); @@ -339,6 +341,30 @@ public class RestController { } } + @PostMapping("/file/upload") + public ResponseEntity uploadFileCheckIfExists( + @RequestParam @NonNull String bucketName, + @RequestParam @NonNull String name, + @RequestParam @NonNull String einfraId, + @RequestBody MultipartFile file) { + try { + verifyVaultCredentials(einfraId); + s3Service.putFileCheckIfExists(bucketName, name, file); + return ResponseEntity.ok("File uploaded."); + } catch (FileExistsException ex) { + return handleFileExistsException(ex); + } catch (S3Exception ex) { + return handleS3Exception(ex, bucketName); + } catch (IOException e) { + return handleIOException( + e, "IOException for bucketName: %s, name: %s.".formatted(bucketName, name)); + } + } + + private void verifyVaultCredentials(String eInfraId) { + credentialsService.verifyVaultCredentials(eInfraId); + } + private ResponseEntity handleS3Exception(S3Exception ex, String bucketName) { if ("NoSuchBucketPolicy".equals(ex.awsErrorDetails().errorCode())) { return ResponseEntity.status(HttpStatus.NOT_FOUND) @@ -366,4 +392,9 @@ public class RestController { return ResponseEntity.badRequest() .body("Access denied. Einfra ID: %s, bucketName: %s".formatted(einfraId, bucketName)); } + + private ResponseEntity handleFileExistsException(FileExistsException ex) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(ex.getMessage()); + } } diff --git a/src/main/java/cz/tul/cxi/DDRcore/exception/FileExistsException.java b/src/main/java/cz/tul/cxi/DDRcore/exception/FileExistsException.java new file mode 100644 index 0000000..48e27f5 --- /dev/null +++ b/src/main/java/cz/tul/cxi/DDRcore/exception/FileExistsException.java @@ -0,0 +1,7 @@ +package cz.tul.cxi.DDRcore.exception; + +public class FileExistsException extends RuntimeException { + public FileExistsException(String bucketName, String key) { + super("File already exists: " + bucketName + "/" + key); + } +} diff --git a/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java b/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java index c375d6a..94fb06d 100644 --- a/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java +++ b/src/main/java/cz/tul/cxi/DDRcore/service/S3Service.java @@ -10,6 +10,7 @@ import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import cz.tul.cxi.DDRcore.exception.FileExistsException; import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.JsonProcessingException; @@ -397,7 +398,15 @@ public class S3Service { s3Component.updateBucketPolicy(bucketName, policyJson); } - public void putFile(String bucketName, String name, MultipartFile image) throws IOException, S3Exception { - s3Component.putFile(bucketName, name, image); + public void putFile(String bucketName, String name, MultipartFile file) throws IOException, S3Exception { + s3Component.putFile(bucketName, name, file); + } + + public void putFileCheckIfExists(String bucketName, String name, MultipartFile file) throws IOException, S3Exception { + if(s3Component.fileExists(bucketName, name)) { + throw new FileExistsException(bucketName, name); + } + + s3Component.putFile(bucketName, name, file); } } -- GitLab