From cd46c0d06e5b78b58c1b4cf43d1990ad54fe5bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Cardiel=20Ferrero?= Date: Mon, 10 Jun 2024 19:08:24 +0200 Subject: [PATCH 1/2] feat(api, engine, frontend, webapp, static): allow HTTP connection on upload media According the feature requested in https://gitlab.com/isard/isardvdi/-/issues/675, new variable `EINGINE_ALLOW_UNSECURED_CONNECTIONS` was introduced in the "engine" section in `isardvdi.cfg` config file to allow unsecure connections (http instead force use https) to upload media from local servers (e.g: python3 -m http.server) without too much restrictions. This new variable has impact in few places; not only engine, also VueJS frontend, Flask frontend and API to validate request. --- api/src/api/views/MediaViews.py | 16 +++++++++++----- docker-compose-parts/api.yml | 3 ++- docker-compose-parts/engine.yml | 1 + docker-compose-parts/static.devel.yml | 1 + docker-compose-parts/webapp.yml | 1 + .../engine/services/threads/download_thread.py | 4 +++- isardvdi.cfg.example | 4 ++++ old-frontend/src/pages/MediaNew.vue | 6 ++++-- .../templates/admin/pages/media_modals.html | 13 +++++++++++++ webapp/webapp/webapp/views/AdminViews.py | 4 ++++ 10 files changed, 44 insertions(+), 9 deletions(-) diff --git a/api/src/api/views/MediaViews.py b/api/src/api/views/MediaViews.py index 9c5291435f..697ae6c3d9 100644 --- a/api/src/api/views/MediaViews.py +++ b/api/src/api/views/MediaViews.py @@ -1,4 +1,5 @@ import json +import os import re import time import traceback @@ -67,11 +68,16 @@ def api_v3_admin_media_insert(payload): traceback.format_exc(), ) url = urlparse(data["url"]) - if url.scheme != "https" or not re.compile( - r"^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|" - r"([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|" - r"([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\." - r"([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$" + allow_unsecured_connections = ( + os.environ.get("ENGINE_ALLOW_UNSECURED_CONNECTIONS", "") == "true" + ) + connection_conditions = ( + ["http", "https"] if allow_unsecured_connections else ["https"] + ) + + if url.scheme not in connection_conditions or not re.compile( + r"^(([a-zA-Z0-9-]{1,63})|(([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9])?\." + r"[a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z]{2,3}))(:[0-9]{1,5})?$" ).match(url.netloc): raise Error( "bad_request", diff --git a/docker-compose-parts/api.yml b/docker-compose-parts/api.yml index 054bff11dd..81be36c279 100644 --- a/docker-compose-parts/api.yml +++ b/docker-compose-parts/api.yml @@ -67,4 +67,5 @@ services: AUTHENTICATION_AUTHENTICATION_GOOGLE_ENABLED: BASTION_ENABLED: BASTION_SSH_PORT: - OPERATIONS_API_ENABLED: \ No newline at end of file + OPERATIONS_API_ENABLED: + ENGINE_ALLOW_UNSECURED_CONNECTIONS: diff --git a/docker-compose-parts/engine.yml b/docker-compose-parts/engine.yml index c947a0858c..ffb6f209ef 100644 --- a/docker-compose-parts/engine.yml +++ b/docker-compose-parts/engine.yml @@ -27,6 +27,7 @@ services: API_ISARDVDI_SECRET: DOMAIN: ENGINE_DISK_BALANCER: + ENGINE_ALLOW_UNSECURED_CONNECTIONS: ENGINE_HYPER_BALANCER: GPU_FAKE: LOG_LEVEL: diff --git a/docker-compose-parts/static.devel.yml b/docker-compose-parts/static.devel.yml index 329ceae156..63ac780330 100644 --- a/docker-compose-parts/static.devel.yml +++ b/docker-compose-parts/static.devel.yml @@ -35,6 +35,7 @@ services: working_dir: /usr/local/src/isard-old-frontend/ environment: CYPRESS_CACHE_FOLDER: /usr/local/src/isard-old-frontend/node_modules/.cache/cypress + VUE_APP_ENGINE_ALLOW_UNSECURED_CONNECTIONS: ${ENGINE_ALLOW_UNSECURED_CONNECTIONS-false} command: sh -c "yarn install && yarn serve" volumes: - ${BUILD_ROOT_PATH}/old-frontend/:/usr/local/src/isard-old-frontend/ diff --git a/docker-compose-parts/webapp.yml b/docker-compose-parts/webapp.yml index 508dea80ec..304ec64630 100644 --- a/docker-compose-parts/webapp.yml +++ b/docker-compose-parts/webapp.yml @@ -22,3 +22,4 @@ services: LOG_LEVEL: GRAFANA_WEBAPP_URL: OPERATIONS_API_ENABLED: + ENGINE_ALLOW_UNSECURED_CONNECTIONS: diff --git a/engine/engine/engine/services/threads/download_thread.py b/engine/engine/engine/services/threads/download_thread.py index 07dd970f47..63fcea8f7b 100644 --- a/engine/engine/engine/services/threads/download_thread.py +++ b/engine/engine/engine/services/threads/download_thread.py @@ -50,7 +50,9 @@ from engine.services.log import logs from isardvdi_common.default_storage_pool import DEFAULT_STORAGE_POOL_ID from rethinkdb import r -URL_DOWNLOAD_INSECURE_SSL = True +URL_DOWNLOAD_INSECURE_SSL = ( + os.environ.get("ENGINE_ALLOW_UNSECURED_CONNECTIONS", "") == "true" +) TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD = 10 FILETYPES_MEDIA_OK = ["application/x-iso9660-image"] diff --git a/isardvdi.cfg.example b/isardvdi.cfg.example index a3fab4c693..ff1fd02a14 100644 --- a/isardvdi.cfg.example +++ b/isardvdi.cfg.example @@ -398,6 +398,10 @@ HAPROXY_LOGGING=. ## ----- Disk operations balancer #ENGINE_DISK_BALANCER=less_cpu +# ------ ENGINE UPLOAD MEDIA --------------------------------------------------- +## Allow unsecured connection to upload media files +#ENGINE_ALLOW_UNSECURED_CONNECTIONS=false + # ------ ENGINE TELEGRAM --------------------------------------------------- ## Will notify this bot/chat if system not fully operational #TELEGRAM_ENGINE_TOKEN= diff --git a/old-frontend/src/pages/MediaNew.vue b/old-frontend/src/pages/MediaNew.vue index b57de27080..5b0d5e88e1 100644 --- a/old-frontend/src/pages/MediaNew.vue +++ b/old-frontend/src/pages/MediaNew.vue @@ -162,7 +162,8 @@ import { required, minLength, maxLength, url } from '@vuelidate/validators' import { map } from 'lodash' import i18n from '@/i18n' const inputFormat = value => /^[-_àèìòùáéíóúñçÀÈÌÒÙÁÉÍÓÚÑÇ .a-zA-Z0-9]+$/.test(value) -const URLFormat = value => /^https:\/\/[a-zA-Z0-9.-_~:/?#[\]@!$&'()*+,;=%]+(?:\/[a-zA-Z0-9.-_~:/?#[\]@!$&'()*+,;=%]+)*(?:\/)?[^\s?#]+(?:\?.*)?$/.test(value) +const URLFormatSecured = value => /^https:\/\/[a-zA-Z0-9.-_~:/?#[\]@!$&'()*+,;=%]+(?:\/[a-zA-Z0-9.-_~:/?#[\]@!$&'()*+,;=%]+)*(?:\/)?[^\s?#]+(?:\?.*)?$/.test(value) +const URLFormatUnsecured = value => /^https?:\/\/[a-zA-Z0-9.-_~:/?#[\]@!$&'()*+,;=%]+(?:\/[a-zA-Z0-9.-_~:/?#[\]@!$&'()*+,;=%]+)*(?:\/)?[^\s?#]+(?:\?.*)?$/.test(value) export default { components: { @@ -187,6 +188,7 @@ export default { const selectedGroups = computed(() => $store.getters.getSelectedGroups) const usersChecked = computed(() => $store.getters.getUsersChecked) const selectedUsers = computed(() => $store.getters.getSelectedUsers) + const allowUnsecured = process.env.VUE_APP_ENGINE_ALLOW_UNSECURED_CONNECTIONS === 'true' const setMediaName = () => { if (name.value === '') { @@ -199,7 +201,7 @@ export default { mediaUrl: { required, url, - URLFormat + URLFormat: allowUnsecured ? URLFormatUnsecured : URLFormatSecured }, type: { required diff --git a/webapp/webapp/webapp/templates/admin/pages/media_modals.html b/webapp/webapp/webapp/templates/admin/pages/media_modals.html index 557d3c920a..d2bae817a8 100644 --- a/webapp/webapp/webapp/templates/admin/pages/media_modals.html +++ b/webapp/webapp/webapp/templates/admin/pages/media_modals.html @@ -21,15 +21,28 @@
+ {% if allow_unsecured_connections %} + + {% else %} + {% endif %}
diff --git a/webapp/webapp/webapp/views/AdminViews.py b/webapp/webapp/webapp/views/AdminViews.py index 5eaaf9dc9d..49569548ce 100644 --- a/webapp/webapp/webapp/views/AdminViews.py +++ b/webapp/webapp/webapp/views/AdminViews.py @@ -30,6 +30,9 @@ from ..auth.authentication import * from ..lib.log import * from .decorators import isAdmin, isAdminManager, maintenance +allow_unsecured_connections = ( + os.environ.get("ENGINE_ALLOW_UNSECURED_CONNECTIONS", "") == "true" +) monitor_host = os.getenv("GRAFANA_WEBAPP_URL") if not monitor_host: monitor_host = f'https://{os.getenv("DOMAIN", "localhost")}/monitor' @@ -296,6 +299,7 @@ def admin_media(): "admin/pages/media.html", nav="Media", title="Media", + allow_unsecured_connections=allow_unsecured_connections, ) -- GitLab From d06d9dde8ae9302a63548500fdeb830531c0e47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Cardiel=20Ferrero?= Date: Tue, 2 Sep 2025 07:52:53 +0200 Subject: [PATCH 2/2] fix: Included default value commented in config file and possible values explained --- isardvdi.cfg.example | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/isardvdi.cfg.example b/isardvdi.cfg.example index ff1fd02a14..b8ea54d925 100644 --- a/isardvdi.cfg.example +++ b/isardvdi.cfg.example @@ -399,7 +399,10 @@ HAPROXY_LOGGING=. #ENGINE_DISK_BALANCER=less_cpu # ------ ENGINE UPLOAD MEDIA --------------------------------------------------- -## Allow unsecured connection to upload media files +## Allow unsecured connections to upload media files +## Values: (first value is the default one) +## - false: only allow https connections +## - true: allow http connections (recommended only local/testing/dev) #ENGINE_ALLOW_UNSECURED_CONNECTIONS=false # ------ ENGINE TELEGRAM --------------------------------------------------- -- GitLab