From 7ccf7a73f1c47a8c03712c39f7c591e837cf6d08 Mon Sep 17 00:00:00 2001 From: Alex Nordlund Date: Wed, 10 Aug 2022 17:12:58 +0200 Subject: [PATCH] Fix broken dependencies in helm chart and allow using existing secrets in the chart (#18941) * Add ability to specify an existing Secret (#18139) Closes #18139 * Allow using secrets with external postgres * Upgrade CronJob to batch/v1 * Allow using redis.auth.existingSecret * Helmignore mastodon-*.tgz for easy local development * Upgrade helm dependencies * Upgrade postgresql to 11 * Allow putting SMTP password into a secret * Add optional login to SMTP secret This to allow setting LOGIN either in values.yaml or in the secret. * Switch to bitnami charts full archive This prevents older versions from disappearing, see https://github.com/bitnami/charts/issues/10539 for full context. Co-authored-by: Ted Tramonte --- chart/.helmignore | 1 + chart/Chart.lock | 16 ++-- chart/Chart.yaml | 14 +-- chart/templates/_helpers.tpl | 50 +++++++++++ chart/templates/configmap-env.yaml | 6 +- chart/templates/cronjob-media-remove.yaml | 14 ++- chart/templates/deployment-sidekiq.yaml | 25 ++++-- chart/templates/deployment-streaming.yaml | 10 +-- chart/templates/deployment-web.yaml | 12 +-- chart/templates/job-assets-precompile.yaml | 12 +-- chart/templates/job-chewy-upgrade.yaml | 12 +-- chart/templates/job-create-admin.yaml | 12 +-- chart/templates/job-db-migrate.yaml | 12 +-- chart/templates/secrets.yaml | 12 ++- chart/values.yaml | 100 ++++++++++++--------- 15 files changed, 183 insertions(+), 125 deletions(-) diff --git a/chart/.helmignore b/chart/.helmignore index 0e8a0eb36f..886747ed0b 100644 --- a/chart/.helmignore +++ b/chart/.helmignore @@ -21,3 +21,4 @@ .idea/ *.tmproj .vscode/ +mastodon-*.tgz diff --git a/chart/Chart.lock b/chart/Chart.lock index d74e7570ce..961e4fa80f 100644 --- a/chart/Chart.lock +++ b/chart/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: elasticsearch - repository: https://charts.bitnami.com/bitnami - version: 15.10.3 + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + version: 19.0.1 - name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 8.10.14 + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + version: 11.1.3 - name: redis - repository: https://charts.bitnami.com/bitnami - version: 10.9.0 -digest: sha256:f5c57108f7768fd16391c1a050991c7809f84a640cca308d7d24d87379d04000 -generated: "2021-08-05T08:01:01.457727804Z" + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + version: 16.13.2 +digest: sha256:17ea58a3264aa22faff18215c4269f47dabae956d0df273c684972f356416193 +generated: "2022-08-08T21:44:18.0195364+02:00" diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 1687132acb..b1138b594f 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.1 +version: 2.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,13 +24,13 @@ appVersion: 3.3.0 dependencies: - name: elasticsearch - version: 15.10.3 - repository: https://charts.bitnami.com/bitnami + version: 19.0.1 + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami condition: elasticsearch.enabled - name: postgresql - version: 8.10.14 - repository: https://charts.bitnami.com/bitnami + version: 11.1.3 + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami condition: postgresql.enabled - name: redis - version: 10.9.0 - repository: https://charts.bitnami.com/bitnami + version: 16.13.2 + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 5814a31203..71bb002ef2 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -77,3 +77,53 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- define "mastodon.postgresql.fullname" -}} {{- printf "%s-%s" .Release.Name "postgresql" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Get the mastodon secret. +*/}} +{{- define "mastodon.secretName" -}} +{{- if .Values.mastodon.secrets.existingSecret }} + {{- printf "%s" (tpl .Values.mastodon.secrets.existingSecret $) -}} +{{- else -}} + {{- printf "%s" (include "common.names.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the postgresql secret. +*/}} +{{- define "mastodon.postgresql.secretName" -}} +{{- if (and (or .Values.postgresql.enabled .Values.postgresql.postgresqlHostname) .Values.postgresql.auth.existingSecret) }} + {{- printf "%s" (tpl .Values.postgresql.auth.existingSecret $) -}} +{{- else if .Values.postgresql.enabled -}} + {{- printf "%s-postgresql" (tpl .Release.Name $) -}} +{{- else -}} + {{- printf "%s" (include "common.names.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the redis secret. +*/}} +{{- define "mastodon.redis.secretName" -}} +{{- if .Values.redis.auth.existingSecret }} + {{- printf "%s" (tpl .Values.redis.auth.existingSecret $) -}} +{{- else if .Values.redis.existingSecret }} + {{- printf "%s" (tpl .Values.redis.existingSecret $) -}} +{{- else -}} + {{- printf "%s-redis" (tpl .Release.Name $) -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a mastodon secret object should be created +*/}} +{{- define "mastodon.createSecret" -}} +{{- if (or + (and .Values.mastodon.s3.enabled (not .Values.mastodon.s3.existingSecret)) + (not .Values.mastodon.secrets.existingSecret ) + (and (not .Values.postgresql.enabled) (not .Values.postgresql.auth.existingSecret)) + ) -}} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/chart/templates/configmap-env.yaml b/chart/templates/configmap-env.yaml index aa242684ff..f988477d91 100644 --- a/chart/templates/configmap-env.yaml +++ b/chart/templates/configmap-env.yaml @@ -10,14 +10,14 @@ data: {{- else }} DB_HOST: {{ .Values.postgresql.postgresqlHostname }} {{- end }} - DB_NAME: {{ .Values.postgresql.postgresqlDatabase }} + DB_NAME: {{ .Values.postgresql.auth.database }} DB_POOL: {{ .Values.mastodon.sidekiq.concurrency | quote }} DB_PORT: "5432" - DB_USER: {{ .Values.postgresql.postgresqlUsername }} + DB_USER: {{ .Values.postgresql.auth.username }} DEFAULT_LOCALE: {{ .Values.mastodon.locale }} {{- if .Values.elasticsearch.enabled }} ES_ENABLED: "true" - ES_HOST: {{ template "mastodon.elasticsearch.fullname" . }}-master + ES_HOST: {{ template "mastodon.elasticsearch.fullname" . }}-master-hl ES_PORT: "9200" {{- end }} LOCAL_DOMAIN: {{ .Values.mastodon.local_domain }} diff --git a/chart/templates/cronjob-media-remove.yaml b/chart/templates/cronjob-media-remove.yaml index 3d6e25cc68..726e100cf8 100644 --- a/chart/templates/cronjob-media-remove.yaml +++ b/chart/templates/cronjob-media-remove.yaml @@ -1,5 +1,5 @@ {{ if .Values.mastodon.cron.removeMedia.enabled }} -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: {{ include "mastodon.fullname" . }}-media-remove @@ -49,21 +49,17 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} diff --git a/chart/templates/deployment-sidekiq.yaml b/chart/templates/deployment-sidekiq.yaml index baf6c2b2dd..f1809bd858 100644 --- a/chart/templates/deployment-sidekiq.yaml +++ b/chart/templates/deployment-sidekiq.yaml @@ -70,22 +70,31 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password + {{- if .Values.mastodon.smtp.existingSecret }} + - name: "SMTP_LOGIN" + valueFrom: + secretKeyRef: + name: {{ .Values.mastodon.smtp.existingSecret }} + key: login + optional: true + - name: "SMTP_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ .Values.mastodon.smtp.existingSecret }} + key: password + {{- end -}} {{- if (not .Values.mastodon.s3.enabled) }} volumeMounts: - name: assets diff --git a/chart/templates/deployment-streaming.yaml b/chart/templates/deployment-streaming.yaml index b332b686ac..12203a530b 100644 --- a/chart/templates/deployment-streaming.yaml +++ b/chart/templates/deployment-streaming.yaml @@ -43,16 +43,12 @@ spec: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.streaming.port | quote }} diff --git a/chart/templates/deployment-web.yaml b/chart/templates/deployment-web.yaml index 8b8bb4f29c..5e22ca5393 100644 --- a/chart/templates/deployment-web.yaml +++ b/chart/templates/deployment-web.yaml @@ -56,21 +56,17 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} diff --git a/chart/templates/job-assets-precompile.yaml b/chart/templates/job-assets-precompile.yaml index 825a7e9160..4aa8d1407c 100644 --- a/chart/templates/job-assets-precompile.yaml +++ b/chart/templates/job-assets-precompile.yaml @@ -50,21 +50,17 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} diff --git a/chart/templates/job-chewy-upgrade.yaml b/chart/templates/job-chewy-upgrade.yaml index cc68a3385a..16b4f75a7d 100644 --- a/chart/templates/job-chewy-upgrade.yaml +++ b/chart/templates/job-chewy-upgrade.yaml @@ -51,21 +51,17 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} diff --git a/chart/templates/job-create-admin.yaml b/chart/templates/job-create-admin.yaml index ffb8bb0590..486c0c3572 100644 --- a/chart/templates/job-create-admin.yaml +++ b/chart/templates/job-create-admin.yaml @@ -56,21 +56,17 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} diff --git a/chart/templates/job-db-migrate.yaml b/chart/templates/job-db-migrate.yaml index 72f910e3b2..41ece64a2a 100644 --- a/chart/templates/job-db-migrate.yaml +++ b/chart/templates/job-db-migrate.yaml @@ -50,21 +50,17 @@ spec: - configMapRef: name: {{ include "mastodon.fullname" . }}-env - secretRef: - name: {{ template "mastodon.fullname" . }} + name: {{ template "mastodon.secretName" . }} env: - name: "DB_PASS" valueFrom: secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ .Release.Name }}-postgresql - {{- else }} - name: {{ template "mastodon.fullname" . }} - {{- end }} - key: postgresql-password + name: {{ template "mastodon.postgresql.secretName" . }} + key: password - name: "REDIS_PASSWORD" valueFrom: secretKeyRef: - name: {{ .Release.Name }}-redis + name: {{ template "mastodon.redis.secretName" . }} key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml index 0452a8ae1e..135d5b61a4 100644 --- a/chart/templates/secrets.yaml +++ b/chart/templates/secrets.yaml @@ -1,3 +1,4 @@ +{{- if (include "mastodon.createSecret" .) }} apiVersion: v1 kind: Secret metadata: @@ -7,9 +8,12 @@ metadata: type: Opaque data: {{- if .Values.mastodon.s3.enabled }} + {{- if not .Values.mastodon.s3.existingSecret }} AWS_ACCESS_KEY_ID: "{{ .Values.mastodon.s3.access_key | b64enc }}" AWS_SECRET_ACCESS_KEY: "{{ .Values.mastodon.s3.access_secret | b64enc }}" {{- end }} + {{- end }} + {{- if not .Values.mastodon.secrets.existingSecret }} {{- if not (empty .Values.mastodon.secrets.secret_key_base) }} SECRET_KEY_BASE: "{{ .Values.mastodon.secrets.secret_key_base | b64enc }}" {{- else }} @@ -30,6 +34,10 @@ data: {{- else }} VAPID_PUBLIC_KEY: {{ required "vapid.public_key is required" .Values.mastodon.secrets.vapid.public_key }} {{- end }} - {{- if not .Values.postgresql.enabled }} - postgresql-password: "{{ .Values.postgresql.postgresqlPassword | b64enc }}" {{- end }} + {{- if not .Values.postgresql.enabled }} + {{- if not .Values.postgresql.auth.existingSecret }} + postgresql-password: "{{ .Values.postgresql.auth.password | b64enc }}" + {{- end }} + {{- end }} +{{- end -}} diff --git a/chart/values.yaml b/chart/values.yaml index 2cfa3484b4..bd723567f0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -48,6 +48,9 @@ mastodon: enabled: false access_key: "" access_secret: "" + # you can also specify the name of an existing Secret + # with keys AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + existingSecret: "" bucket: "" endpoint: https://us-east-1.linodeobjects.com hostname: us-east-1.linodeobjects.com @@ -61,6 +64,10 @@ mastodon: vapid: private_key: "" public_key: "" + # you can also specify the name of an existing Secret + # with keys SECRET_KEY_BASE and OTP_SECRET and + # VAPID_PRIVATE_KEY and VAPID_PUBLIC_KEY + existingSecret: "" sidekiq: concurrency: 25 smtp: @@ -70,13 +77,16 @@ mastodon: domain: enable_starttls_auto: true from_address: notifications@example.com - login: openssl_verify_mode: peer - password: port: 587 reply_to: server: smtp.mailgun.org tls: false + login: + password: + # you can also specify the name of an existing Secret + # with the keys login and password + existingSecret: streaming: port: 4000 # this should be set manually since os.cpus() returns the number of CPUs on @@ -127,18 +137,26 @@ postgresql: # must match those of that external postgres instance enabled: true # postgresqlHostname: preexisting-postgresql - postgresqlDatabase: mastodon_production - # you must set a password; the password generated by the postgresql chart will - # be rotated on each upgrade: - # https://github.com/bitnami/charts/tree/master/bitnami/postgresql#upgrade - postgresqlPassword: "" - postgresqlUsername: postgres + auth: + database: mastodon_production + username: postgres + # you must set a password; the password generated by the postgresql chart will + # be rotated on each upgrade: + # https://github.com/bitnami/charts/tree/master/bitnami/postgresql#upgrade + password: "" + # you can also specify the name of an existing Secret + # with a key of postgres-password set to the password you want + existingSecret: "" # https://github.com/bitnami/charts/tree/master/bitnami/redis#parameters redis: # you must set a password; the password generated by the redis chart will be # rotated on each upgrade: password: "" + # you can also specify the name of an existing Secret + # with a key of redis-password set to the password you want + # auth: + # existingSecret: "" service: type: ClusterIP @@ -157,45 +175,45 @@ externalAuth: # client_secret: SECRETKEY # redirect_uri: https://example.com/auth/auth/openid_connect/callback # assume_email_is_verified: true - # client_auth_method: - # response_type: - # response_mode: - # display: - # prompt: - # send_nonce: - # send_scope_to_token_endpoint: - # idp_logout_redirect_uri: - # http_scheme: - # host: - # port: - # jwks_uri: - # auth_endpoint: - # token_endpoint: - # user_info_endpoint: - # end_session_endpoint: + # client_auth_method: + # response_type: + # response_mode: + # display: + # prompt: + # send_nonce: + # send_scope_to_token_endpoint: + # idp_logout_redirect_uri: + # http_scheme: + # host: + # port: + # jwks_uri: + # auth_endpoint: + # token_endpoint: + # user_info_endpoint: + # end_session_endpoint: saml: enabled: false # acs_url: http://mastodon.example.com/auth/auth/saml/callback # issuer: mastodon # idp_sso_target_url: https://login.example.com/auth/realms/example/protocol/saml # idp_cert: '-----BEGIN CERTIFICATE-----[your_cert_content]-----END CERTIFICATE-----' - # idp_cert_fingerprint: + # idp_cert_fingerprint: # name_identifier_format: urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified - # cert: - # private_key: + # cert: + # private_key: # want_assertion_signed: true # want_assertion_encrypted: true # assume_email_is_verified: true # uid_attribute: "urn:oid:0.9.2342.19200300.100.1.1" - # attributes_statements: + # attributes_statements: # uid: "urn:oid:0.9.2342.19200300.100.1.1" # email: "urn:oid:1.3.6.1.4.1.5923.1.1.1.6" # full_name: "urn:oid:2.16.840.1.113730.3.1.241" # first_name: "urn:oid:2.5.4.42" # last_name: "urn:oid:2.5.4.4" - # verified: - # verified_email: - oauth_global: + # verified: + # verified_email: + oauth_global: # Force redirect local login to CAS. Does not function with SAML or LDAP. oauth_redirect_at_sign_in: false cas: @@ -204,15 +222,15 @@ externalAuth: # host: sso.myserver.com # port: 443 # ssl: true - # validate_url: - # callback_url: - # logout_url: - # login_url: + # validate_url: + # callback_url: + # logout_url: + # login_url: # uid_field: 'user' - # ca_path: + # ca_path: # disable_ssl_verification: false # assume_email_is_verified: true - # keys: + # keys: # uid: 'user' # name: 'name' # email: 'email' @@ -222,7 +240,7 @@ externalAuth: # location: 'location' # image: 'image' # phone: 'phone' - pam: + pam: enabled: false # email_domain: example.com # default_service: rpam @@ -232,9 +250,9 @@ externalAuth: # host: myservice.namespace.svc # port: 389 # method: simple_tls - # base: - # bind_on: - # password: + # base: + # bind_on: + # password: # uid: cn # mail: mail # search_filter: "(|(%{uid}=%{email})(%{mail}=%{email}))"