From 72ceda6108b2de9def82e63cb086abc5d37e2705 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Wed, 16 Mar 2022 19:54:52 +0000 Subject: [PATCH 1/6] Expose testslowimport for the registry --- charts/registry/templates/_migration.tpl | 6 +++++- charts/registry/values.yaml | 1 + templates/NOTES.txt | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/charts/registry/templates/_migration.tpl b/charts/registry/templates/_migration.tpl index 666a050b21..26678ded30 100644 --- a/charts/registry/templates/_migration.tpl +++ b/charts/registry/templates/_migration.tpl @@ -6,7 +6,11 @@ migration: enabled: {{ .Values.migration.enabled | eq true }} {{- if .Values.migration.disablemirrorfs }} disablemirrorfs: true -{{- end }} +{{- end -}} + {{/* This value is used for testing purposes only and will be removed soon + TODO: https://gitlab.com/gitlab-org/container-registry/-/issues/616 + */}} + testslowimport: {{ .Values.migration.testslowimport }} {{- if .Values.migration.rootdirectory }} rootdirectory: {{ .Values.migration.rootdirectory }} {{- end }} diff --git a/charts/registry/values.yaml b/charts/registry/values.yaml index 342480c4c3..27b7422f70 100644 --- a/charts/registry/values.yaml +++ b/charts/registry/values.yaml @@ -350,6 +350,7 @@ database: migration: enabled: false disablemirrorfs: false + testslowimport: "0s" # rootdirectory: # tagconcurrency: 1 # maxconcurrentimports: 1 diff --git a/templates/NOTES.txt b/templates/NOTES.txt index a3421b540c..f356bd50a2 100644 --- a/templates/NOTES.txt +++ b/templates/NOTES.txt @@ -14,6 +14,11 @@ WARNING: The Container Registry filesystem metadata has been disabled. Please no {{/* If the Container Registry migration mode is enabled, note it is not production ready */}} {{- if eq .Values.registry.migration.enabled true }} WARNING: The Container Registry migration mode has been enabled. Please note that it is an experimental feature and must not be used in production. + +{{- if ne .Values.registry.migration.testslowimport "0s" }} +WARNING: The Container Registry test slow import has been enabled. Please note that it is an experimental feature used for testing purposes only and must not be used in production. +{{- end }} + {{- end }} {{/* If the Container Registry online garbage collection is enabled, note it is not production ready */}} -- GitLab From 26dfa2fc0fb195af66b98008658a5e28f3298f10 Mon Sep 17 00:00:00 2001 From: Gerard Hickey Date: Fri, 11 Mar 2022 20:12:32 -0800 Subject: [PATCH 2/6] Updated PostgreSQL checkConfig passing rspec tests --- templates/_checkConfig_postgresql.tpl | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/templates/_checkConfig_postgresql.tpl b/templates/_checkConfig_postgresql.tpl index 0a2060b190..b0f86fa749 100644 --- a/templates/_checkConfig_postgresql.tpl +++ b/templates/_checkConfig_postgresql.tpl @@ -24,7 +24,15 @@ Ensure that if `psql.password.useSecret` is set to false, a path to the password {{- $errorMsg := list -}} {{- $subcharts := pick .Values.gitlab "geo-logcursor" "gitlab-exporter" "migrations" "sidekiq" "toolbox" "webservice" -}} {{- range $name, $sub := $subcharts -}} -{{- $useSecret := include "gitlab.boolean.local" (dict "local" (pluck "useSecret" (index $sub "psql" "password") | first) "global" $.Values.global.psql.password.useSecret "default" true) -}} +{{- $main_useSecret := false -}} +{{- if hasKey $.Values.global.psql "main" -}} +{{- if hasKey $.Values.global.psql.main "password" -}} +{{- if hasKey $.Values.global.psql.main.password "useSecret" -}} +{{- $main_useSecret = default $.Values.global.psql.main.password.useSecret false -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- $useSecret := include "gitlab.boolean.local" (dict "local" (pluck "useSecret" (index $sub "psql" "password") | first) "main" $main_useSecret "global" $.Values.global.psql.password.useSecret "default" true) -}} {{- if and (not $useSecret) (not (pluck "file" (index $sub "psql" "password") ($.Values.global.psql.password) | first)) -}} {{- $errorMsg = append $errorMsg (printf "%s: If `psql.password.useSecret` is set to false, you must specify a value for `psql.password.file`." $name) -}} {{- end -}} @@ -46,7 +54,15 @@ postgresql: Ensure that `postgresql.install: false` when `global.psql.load_balancing` defined */}} {{- define "gitlab.checkConfig.database.externalLoadBalancing" -}} -{{- if hasKey .Values.global.psql "load_balancing" -}} +{{- $load_balancing := dict -}} +{{- if hasKey .Values.global.psql "main" -}} +{{- if hasKey .Values.global.psql.main "load_balancing" -}} +{{- $load_balancing = .Values.global.psql.main.load_balancing -}} +{{- end -}} +{{- else if hasKey .Values.global.psql "load_balancing" -}} +{{- $load_balancing = .Values.global.psql.load_balancing -}} +{{- end -}} +{{- if $load_balancing -}} {{- with .Values.global.psql.load_balancing -}} {{- if and $.Values.postgresql.install (kindIs "map" .) }} postgresql: -- GitLab From 024e07b00f6a4a06773da020d50fb46306350e9f Mon Sep 17 00:00:00 2001 From: Gerard Hickey Date: Tue, 15 Mar 2022 14:00:49 -0700 Subject: [PATCH 3/6] Update of PostgreSQL ssl checks --- charts/gitlab/templates/_postgresql.tpl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/gitlab/templates/_postgresql.tpl b/charts/gitlab/templates/_postgresql.tpl index 0f6074e587..2a7df90e20 100644 --- a/charts/gitlab/templates/_postgresql.tpl +++ b/charts/gitlab/templates/_postgresql.tpl @@ -3,7 +3,7 @@ Returns parts for a Gitlab configuration to setup a mutual TLS connection with the PostgreSQL database. */}} {{- define "gitlab.psql.ssl.config" -}} -{{- if .Values.global.psql.ssl }} +{{- if or .ssl .Values.global.psql.ssl }} sslmode: verify-ca sslrootcert: '/etc/gitlab/postgres/ssl/server-ca.pem' sslcert: '/etc/gitlab/postgres/ssl/client-certificate.pem' @@ -16,19 +16,19 @@ Returns volume definition of a secret containing information required for a mutual TLS connection. */}} {{- define "gitlab.psql.ssl.volume" -}} -{{- if .Values.global.psql.ssl }} +{{- if or .ssl .Values.global.psql.ssl }} - name: postgresql-ssl-secrets projected: defaultMode: 400 sources: - secret: - name: {{ .Values.global.psql.ssl.secret | required "Missing required secret containing SQL SSL certificates and keys. Make sure to set `global.psql.ssl.secret`" }} + name: {{ default .ssl.secret .Values.global.psql.ssl.secret | required "Missing required secret containing SQL SSL certificates and keys. Make sure to set `global.psql.ssl.secret` or `.ssl.secret` in database settings" }} items: - - key: {{ .Values.global.psql.ssl.serverCA | required "Missing required key name of SQL server certificate. Make sure to set `global.psql.ssl.serverCA`" }} + - key: {{ default .ssl.serverCA .Values.global.psql.ssl.serverCA | required "Missing required key name of SQL server certificate. Make sure to set `global.psql.ssl.serverCA` or `.ssl.serverCA` in database settings" }} path: server-ca.pem - - key: {{ .Values.global.psql.ssl.clientCertificate | required "Missing required key name of SQL client certificate. Make sure to set `global.psql.ssl.clientCertificate`" }} + - key: {{ default .ssl.clientCertificate .Values.global.psql.ssl.clientCertificate | required "Missing required key name of SQL client certificate. Make sure to set `global.psql.ssl.clientCertificate` or `.ssl.clientCertificate` in database settings" }} path: client-certificate.pem - - key: {{ .Values.global.psql.ssl.clientKey | required "Missing required key name of SQL client key file. Make sure to set `global.psql.ssl.clientKey`" }} + - key: {{ default .ssl.clientKey .Values.global.psql.ssl.clientKey | required "Missing required key name of SQL client key file. Make sure to set `global.psql.ssl.clientKey` or `.ssl.clientKey` in database settings" }} path: client-key.pem {{- end -}} {{- end -}} @@ -37,7 +37,7 @@ a mutual TLS connection. Returns mount definition for the volume mount definition above. */}} {{- define "gitlab.psql.ssl.volumeMount" -}} -{{- if .Values.global.psql.ssl }} +{{- if or .ssl .Values.global.psql.ssl }} - name: postgresql-ssl-secrets mountPath: '/etc/postgresql/ssl/' readOnly: true @@ -50,7 +50,7 @@ container to copy the mutual TLS files to the proper location. Further it sets the permissions correctly. */}} {{- define "gitlab.psql.ssl.initScript" -}} -{{- if .Values.global.psql.ssl }} +{{- if or .ssl .Values.global.psql.ssl }} if [ -d /etc/postgresql/ssl ]; then mkdir -p /${secret_dir}/postgres/ssl cp -v -r -L /etc/postgresql/ssl/* /${secret_dir}/postgres/ssl/ -- GitLab From 3398ef67ac0b9fb51912ca498e1541240790e587 Mon Sep 17 00:00:00 2001 From: Gerard Hickey Date: Thu, 17 Mar 2022 16:30:31 -0700 Subject: [PATCH 4/6] Removed errant context statement --- spec/configuration/database_spec.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/configuration/database_spec.rb b/spec/configuration/database_spec.rb index c2580e85da..91fb3cdd10 100644 --- a/spec/configuration/database_spec.rb +++ b/spec/configuration/database_spec.rb @@ -59,8 +59,6 @@ describe 'Database configuration' do end end - context - describe 'global.psql settings' do context 'when psql.database set globally' do let(:global_values) do -- GitLab From aee2300869e8018efe0d29f44314abd0d72aece5 Mon Sep 17 00:00:00 2001 From: Gerard Hickey Date: Thu, 14 Apr 2022 16:29:18 -0700 Subject: [PATCH 5/6] Added documentation for PostgreSQL data model --- .../gitlab/templates/_databaseDatamodel.tpl | 4 +- doc/development/datamodels.md | 156 ++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 doc/development/datamodels.md diff --git a/charts/gitlab/templates/_databaseDatamodel.tpl b/charts/gitlab/templates/_databaseDatamodel.tpl index 824fd763fb..f64d7b2f2f 100644 --- a/charts/gitlab/templates/_databaseDatamodel.tpl +++ b/charts/gitlab/templates/_databaseDatamodel.tpl @@ -22,6 +22,8 @@ Removed: {{/* database.datamodel.prepare +Also documented in doc/development/datamodels.md with examples. + Result: `.Values.local.psql` contains a fully composed datamodel of psql properties to be passed as the context to other helpers. Which Schema you are in can @@ -32,7 +34,7 @@ How: - ensure `main` exists in both global and local, if not present. - mergeOverwrite `.global.psql` `.global.psql.x` - mergeOverwrite `.psql` `.psql.x` - - build $context dict, with .Release .Values.global.psql .Values.psql + - build $context dict, with .Release .Values.global.psql .Values.psql Example object - local: diff --git a/doc/development/datamodels.md b/doc/development/datamodels.md new file mode 100644 index 0000000000..ca8e01cfa7 --- /dev/null +++ b/doc/development/datamodels.md @@ -0,0 +1,156 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Data model implementations + +This document attempts to document the data model schemas used within the +chart. + +It is important to understand that the data model is located within the context +where it is instantiated, not in `.Values`. + +## PostgreSQL data model + +The PostgreSQL data model is instantiated with the following: + +``` +{{- include "database.datamodel.prepare" . -}} +``` + +This would be needed to be called in any Helm template file that needs to +access the PostgreSQL data model. It will examine the computed `.Values` and +generate the PostgreSQL data model in the current context. + +### Traditional PostgreSQL configuration + +Most installations use a single PostgreSQL database and will have the +configuration specified as: + +```yaml +global: + psql: + host: pg-postgresql.pg.svc + database: gitlabhq_production + username: postgres + preparedStatements: false + password: + secret: gitlab-postgres + key: psql-password + +postgresql: + install: false +``` + +The resulting data model will be generated. Note that `main` is used for +the primary database connection. Additional database connections are +created under additional attributes as shown in the decomposed database +configuration later. + +```yaml +: + local: + psql: + main: + Release: + IsInstall: true + IsUpgrade: false + Name: test + Namespace: kube-public + Revision: 1 + Service: Helm + Schema: main + Values: + global: + psql: + database: gitlabhq_production + host: pg-postgresql.pg.svc + password: + key: psql-password + secret: gitlab-postgres + preparedStatements: false + username: postgres + psql: {} +``` + +### Decomposed PostgreSQL configuration + +If one is using a decomposed database configuration, then the values for +the database configuration will be similar to: + +```yaml +global: + psql: + main: + host: pg-main-postgresql.pg.svc + database: gitlabhq_production + username: postgres + preparedStatements: false + password: + secret: gitlab-main-postgres + key: psql-main-password + ci: + host: pg-ci-postgresql.pg.svc + database: gitlabhq_production + username: postgres + preparedStatements: false + password: + secret: gitlab-ci-postgres + key: psql-ci-password + +postgresql: + install: false +``` + +The resulting data model for a decomposed database configuration is as +follows: + +```yaml +: + local: + psql: + ci: + Release: + IsInstall: true + IsUpgrade: false + Name: test + Namespace: default + Revision: 1 + Service: Helm + Schema: ci + Values: + global: + psql: + database: gitlabhq_production + host: pg-ci-postgresql.pg.svc + password: + key: psql-ci-password + secret: gitlab-ci-postgres + preparedStatements: false + username: postgres + psql: {} + main: + Release: + IsInstall: true + IsUpgrade: false + Name: test + Namespace: default + Revision: 1 + Service: Helm + Schema: main + Values: + global: + psql: + database: gitlabhq_production + host: pg-main-postgresql.pg.svc + password: + key: psql-main-password + secret: gitlab-main-postgres + preparedStatements: false + username: postgres + psql: {} +``` + + -- GitLab From 08da5d74de8c3c83b9ee8c4679bf436d39616ed7 Mon Sep 17 00:00:00 2001 From: Gerard Hickey Date: Tue, 19 Apr 2022 12:13:18 -0700 Subject: [PATCH 6/6] Use dig to interogate the PostgreSQL data model Signed-off-by: Gerard Hickey Changelog: other --- templates/_checkConfig_postgresql.tpl | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/templates/_checkConfig_postgresql.tpl b/templates/_checkConfig_postgresql.tpl index b0f86fa749..b6d1061254 100644 --- a/templates/_checkConfig_postgresql.tpl +++ b/templates/_checkConfig_postgresql.tpl @@ -24,14 +24,7 @@ Ensure that if `psql.password.useSecret` is set to false, a path to the password {{- $errorMsg := list -}} {{- $subcharts := pick .Values.gitlab "geo-logcursor" "gitlab-exporter" "migrations" "sidekiq" "toolbox" "webservice" -}} {{- range $name, $sub := $subcharts -}} -{{- $main_useSecret := false -}} -{{- if hasKey $.Values.global.psql "main" -}} -{{- if hasKey $.Values.global.psql.main "password" -}} -{{- if hasKey $.Values.global.psql.main.password "useSecret" -}} -{{- $main_useSecret = default $.Values.global.psql.main.password.useSecret false -}} -{{- end -}} -{{- end -}} -{{- end -}} +{{- $main_useSecret := dig "main" "password" "useSecret" false $.Values.global.psql -}} {{- $useSecret := include "gitlab.boolean.local" (dict "local" (pluck "useSecret" (index $sub "psql" "password") | first) "main" $main_useSecret "global" $.Values.global.psql.password.useSecret "default" true) -}} {{- if and (not $useSecret) (not (pluck "file" (index $sub "psql" "password") ($.Values.global.psql.password) | first)) -}} {{- $errorMsg = append $errorMsg (printf "%s: If `psql.password.useSecret` is set to false, you must specify a value for `psql.password.file`." $name) -}} @@ -54,14 +47,12 @@ postgresql: Ensure that `postgresql.install: false` when `global.psql.load_balancing` defined */}} {{- define "gitlab.checkConfig.database.externalLoadBalancing" -}} -{{- $load_balancing := dict -}} -{{- if hasKey .Values.global.psql "main" -}} -{{- if hasKey .Values.global.psql.main "load_balancing" -}} -{{- $load_balancing = .Values.global.psql.main.load_balancing -}} -{{- end -}} -{{- else if hasKey .Values.global.psql "load_balancing" -}} -{{- $load_balancing = .Values.global.psql.load_balancing -}} +{{- if hasKey .Values.global.psql "load_balancing" -}} +{{- $load_balancing := .Values.global.psql.load_balancing -}} +{{- else -}} +{{- $load_balancing :=dict -}} {{- end -}} +{{- $load_balancing := dig "main" "load_balancing" $load_balancing .Values.global.psql -}} {{- if $load_balancing -}} {{- with .Values.global.psql.load_balancing -}} {{- if and $.Values.postgresql.install (kindIs "map" .) }} -- GitLab