From eeb1b619b71b580c6aa67caa14f3f39631fd42f0 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Tue, 1 Aug 2023 23:31:14 +0200 Subject: [PATCH] Automate Zitadel IDP configuration (#1006) Add automated zitadel configuration and netbird using a single script run infra files test only when they change or when pushed to main run releases only on changes to related files or when pushed to main push install and automated getting started script to releases page --- .github/workflows/release.yml | 10 + ...inux.yml => test-infrastructure-files.yml} | 34 +- .goreleaser.yaml | 5 + .../getting-started-with-zitadel.sh | 717 ++++++++++++++++++ infrastructure_files/zitadel.sh | 122 --- 5 files changed, 763 insertions(+), 125 deletions(-) rename .github/workflows/{test-docker-compose-linux.yml => test-infrastructure-files.yml} (87%) create mode 100644 infrastructure_files/getting-started-with-zitadel.sh delete mode 100644 infrastructure_files/zitadel.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a3c04f5d..aaae51dde 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,16 @@ on: branches: - main pull_request: + paths: + - 'go.mod' + - 'go.sum' + - '.goreleaser.yml' + - '.goreleaser_ui.yaml' + - '.goreleaser_ui_darwin.yaml' + - '.github/workflows/release.yml' + - 'release_files/**' + - '**/Dockerfile' + - '**/Dockerfile.*' env: SIGN_PIPE_VER: "v0.0.8" diff --git a/.github/workflows/test-docker-compose-linux.yml b/.github/workflows/test-infrastructure-files.yml similarity index 87% rename from .github/workflows/test-docker-compose-linux.yml rename to .github/workflows/test-infrastructure-files.yml index 3910de0f2..920c6f460 100644 --- a/.github/workflows/test-docker-compose-linux.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -1,10 +1,13 @@ -name: Test Docker Compose Linux +name: Test Infrastructure files on: push: branches: - main pull_request: + paths: + - 'infrastructure_files/**' + - '.github/workflows/test-infrastructure-files.yml' concurrency: @@ -12,7 +15,7 @@ concurrency: cancel-in-progress: true jobs: - test: + test-docker-compose: runs-on: ubuntu-latest steps: - name: Install jq @@ -35,7 +38,7 @@ jobs: ${{ runner.os }}-go- - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: cp setup.env run: cp infrastructure_files/tests/setup.env infrastructure_files/ @@ -121,3 +124,28 @@ jobs: count=$(docker compose ps --format json | jq '.[] | select(.Project | contains("infrastructure_files")) | .State' | grep -c running) test $count -eq 4 working-directory: infrastructure_files + + test-getting-started-script: + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get install -y jq + + - name: Checkout code + uses: actions/checkout@v3 + + - name: run script + run: bash -x infrastructure_files/getting-started-with-zitadel.sh + + - name: test Caddy file gen + run: test -f Caddyfile + - name: test docker-compose file gen + run: test -f docker-compose.yml + - name: test management.json file gen + run: test -f management.json + - name: test turnserver.conf file gen + run: test -f turnserver.conf + - name: test zitadel.env file gen + run: test -f zitadel.env + - name: test dashboard.env file gen + run: test -f dashboard.env \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e4dca7478..1d31a949a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -377,3 +377,8 @@ uploads: target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com method: PUT + +release: + extra_files: + - glob: ./infrastructure_files/getting-started-with-zitadel.sh + - glob: ./release_files/install.sh \ No newline at end of file diff --git a/infrastructure_files/getting-started-with-zitadel.sh b/infrastructure_files/getting-started-with-zitadel.sh new file mode 100644 index 000000000..da637dc5d --- /dev/null +++ b/infrastructure_files/getting-started-with-zitadel.sh @@ -0,0 +1,717 @@ +#!/bin/bash + +set -e + +handle_request_command_status() { + PARSED_RESPONSE=$1 + FUNCTION_NAME=$2 + RESPONSE=$3 + if [[ $PARSED_RESPONSE -ne 0 ]]; then + echo "ERROR calling $FUNCTION_NAME:" $(echo "$RESPONSE" | jq -r '.message') > /dev/stderr + exit 1 + fi +} + +handle_zitadel_request_response() { + PARSED_RESPONSE=$1 + FUNCTION_NAME=$2 + RESPONSE=$3 + if [[ $PARSED_RESPONSE == "null" ]]; then + echo "ERROR calling $FUNCTION_NAME:" $(echo "$RESPONSE" | jq -r '.message') > /dev/stderr + exit 1 + fi + sleep 1 +} + +check_docker_compose() { + if command -v docker-compose &> /dev/null + then + echo "docker-compose" + return + fi + if docker compose --help &> /dev/null + then + echo "docker compose" + return + fi + + echo "docker-compose is not installed or not in PATH. Please follow the steps from the official guide: https://docs.docker.com/engine/install/" + exit 1 +} + +check_jq() { + if ! command -v jq &> /dev/null + then + echo "jq is not installed or not in PATH, please install with your package manager. e.g. sudo apt install jq" + exit 1 + fi +} + +wait_crdb() { + set +e + while true; do + if $DOCKER_COMPOSE_COMMAND exec -T crdb curl -sf 'http://localhost:8080/health?ready=1'; then + break + fi + echo -n " ." + sleep 5 + done + echo " done" + set -e +} + +init_crdb() { + echo -n "Initializing crdb " + $DOCKER_COMPOSE_COMMAND up -d crdb + wait_crdb + $DOCKER_COMPOSE_COMMAND exec -T crdb /bin/bash -c "cp /cockroach/certs/* /zitadel-certs/ && cockroach cert create-client --overwrite --certs-dir /zitadel-certs/ --ca-key /zitadel-certs/ca.key zitadel_user && chown -R 1000:1000 /zitadel-certs/" + handle_request_command_status $? "init_crdb failed" "" +} + +get_main_ip_address() { + if [[ "$OSTYPE" == "darwin"* ]]; then + interface=$(route -n get default | grep 'interface:' | awk '{print $2}') + ip_address=$(ifconfig "$interface" | grep 'inet ' | awk '{print $2}') + else + interface=$(ip route | grep default | awk '{print $5}' | head -n 1) + ip_address=$(ip addr show "$interface" | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) + fi + + echo "$ip_address" +} + +wait_pat() { + PAT_PATH=$1 + set +e + while true; do + if [[ -f "$PAT_PATH" ]]; then + break + fi + echo -n " ." + sleep 1 + done + echo " done" + set -e +} + +wait_api() { + INSTANCE_URL=$1 + PAT=$2 + set +e + while true; do + curl -s --fail -o /dev/null "$INSTANCE_URL/auth/v1/users/me" -H "Authorization: Bearer $PAT" + if [[ $? -eq 0 ]]; then + break + fi + echo -n " ." + sleep 1 + done + echo " done" + set -e +} + +create_new_project() { + INSTANCE_URL=$1 + PAT=$2 + PROJECT_NAME="NETBIRD" + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/projects" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{"name": "'"$PROJECT_NAME"'"}' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.id') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_new_project" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_new_application() { + INSTANCE_URL=$1 + PAT=$2 + APPLICATION_NAME=$3 + BASE_REDIRECT_URL1=$4 + BASE_REDIRECT_URL2=$5 + ZITADEL_DEV_MODE=$6 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/projects/$PROJECT_ID/apps/oidc" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "'"$APPLICATION_NAME"'", + "redirectUris": [ + "'"$BASE_REDIRECT_URL1"'", + "'"$BASE_REDIRECT_URL2"'" + ], + "postLogoutRedirectUris": [ + "'"$BASE_REDIRECT_URL1"'" + ], + "RESPONSETypes": [ + "OIDC_RESPONSE_TYPE_CODE" + ], + "grantTypes": [ + "OIDC_GRANT_TYPE_AUTHORIZATION_CODE", + "OIDC_GRANT_TYPE_REFRESH_TOKEN" + ], + "appType": "OIDC_APP_TYPE_USER_AGENT", + "authMethodType": "OIDC_AUTH_METHOD_TYPE_NONE", + "version": "OIDC_VERSION_1_0", + "devMode": '"$ZITADEL_DEV_MODE"', + "accessTokenType": "OIDC_TOKEN_TYPE_JWT", + "accessTokenRoleAssertion": true, + "skipNativeAppSuccessPage": true + }' + ) + + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.clientId') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_new_application" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_service_user() { + INSTANCE_URL=$1 + PAT=$2 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/users/machine" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userName": "netbird-service-account", + "name": "Netbird Service Account", + "description": "Netbird Service Account for IDP management", + "accessTokenType": "ACCESS_TOKEN_TYPE_JWT" + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.userId') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_service_user" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_service_user_secret() { + INSTANCE_URL=$1 + PAT=$2 + USER_ID=$3 + + RESPONSE=$( + curl -sS -X PUT "$INSTANCE_URL/management/v1/users/$USER_ID/secret" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{}' + ) + SERVICE_USER_CLIENT_ID=$(echo "$RESPONSE" | jq -r '.clientId') + handle_zitadel_request_response "$SERVICE_USER_CLIENT_ID" "create_service_user_secret_id" "$RESPONSE" + SERVICE_USER_CLIENT_SECRET=$(echo "$RESPONSE" | jq -r '.clientSecret') + handle_zitadel_request_response "$SERVICE_USER_CLIENT_SECRET" "create_service_user_secret" "$RESPONSE" +} + +add_organization_user_manager() { + INSTANCE_URL=$1 + PAT=$2 + USER_ID=$3 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/orgs/me/members" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "'"$USER_ID"'", + "roles": [ + "ORG_USER_MANAGER" + ] + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.creationDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "add_organization_user_manager" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_admin_user() { + INSTANCE_URL=$1 + PAT=$2 + USERNAME=$3 + PASSWORD=$4 + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/users/human/_import" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userName": "'"$USERNAME"'", + "profile": { + "firstName": "Zitadel", + "lastName": "Admin" + }, + "email": { + "email": "'"$USERNAME"'", + "isEmailVerified": true + }, + "password": "'"$PASSWORD"'", + "passwordChangeRequired": true + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.userId') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_admin_user" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +add_instance_admin() { + INSTANCE_URL=$1 + PAT=$2 + USER_ID=$3 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/admin/v1/members" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "'"$USER_ID"'", + "roles": [ + "IAM_OWNER" + ] + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.creationDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "add_instance_admin" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +delete_auto_service_user() { + INSTANCE_URL=$1 + PAT=$2 + + RESPONSE=$( + curl -sS -X GET "$INSTANCE_URL/auth/v1/users/me" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + ) + USER_ID=$(echo "$RESPONSE" | jq -r '.user.id') + handle_zitadel_request_response "$USER_ID" "delete_auto_service_user_get_user" "$RESPONSE" + + RESPONSE=$( + curl -sS -X DELETE "$INSTANCE_URL/admin/v1/members/$USER_ID" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.changeDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "delete_auto_service_user_remove_instance_permissions" "$RESPONSE" + + RESPONSE=$( + curl -sS -X DELETE "$INSTANCE_URL/management/v1/orgs/me/members/$USER_ID" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.changeDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "delete_auto_service_user_remove_org_permissions" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +init_zitadel() { + echo "Initializing Zitadel IDP" + INSTANCE_URL="$NETBIRD_HTTP_PROTOCOL://$NETBIRD_DOMAIN:$NETBIRD_PORT" + + TOKEN_PATH=./machinekey/zitadel-admin-sa.token + + # shellcheck disable=SC2028 + echo -n "Waiting for Zitadel's PAT to be created " + wait_pat "$TOKEN_PATH" + echo "Reading Zitadel PAT" + PAT=$(cat $TOKEN_PATH) + if [ "$PAT" = "null" ]; then + echo "Failed requesting getting Zitadel PAT" + exit 1 + fi + + # shellcheck disable=SC2028 + echo -n "Waiting for Zitadel to become ready " + wait_api "$INSTANCE_URL" "$PAT" + + # create the zitadel project + echo "Creating new zitadel project" + PROJECT_ID=$(create_new_project "$INSTANCE_URL" "$PAT") + + ZITADEL_DEV_MODE=false + BASE_REDIRECT_URL=$NETBIRD_HTTP_PROTOCOL://$NETBIRD_DOMAIN + if [[ $NETBIRD_HTTP_PROTOCOL == "http" ]]; then + ZITADEL_DEV_MODE=true + fi + + # create zitadel spa applications + echo "Creating new Zitadel SPA Dashboard application" + DASHBOARD_APPLICATION_CLIENT_ID=$(create_new_application "$INSTANCE_URL" "$PAT" "Dashboard" "$BASE_REDIRECT_URL/nb-auth" "$BASE_REDIRECT_URL/nb-silent-auth" "$ZITADEL_DEV_MODE") + + echo "Creating new Zitadel SPA Cli application" + CLI_APPLICATION_CLIENT_ID=$(create_new_application "$INSTANCE_URL" "$PAT" "Cli" "http://localhost:53000/" "http://localhost:54000/" "true") + + MACHINE_USER_ID=$(create_service_user "$INSTANCE_URL" "$PAT") + + SERVICE_USER_CLIENT_ID="null" + SERVICE_USER_CLIENT_SECRET="null" + + create_service_user_secret "$INSTANCE_URL" "$PAT" "$MACHINE_USER_ID" + + DATE=$(add_organization_user_manager "$INSTANCE_URL" "$PAT" "$MACHINE_USER_ID") + + ZITADEL_ADMIN_USERNAME="admin@$NETBIRD_DOMAIN" + ZITADEL_ADMIN_PASSWORD="$(openssl rand -base64 32 | sed 's/=//g')@" + + HUMAN_USER_ID=$(create_admin_user "$INSTANCE_URL" "$PAT" "$ZITADEL_ADMIN_USERNAME" "$ZITADEL_ADMIN_PASSWORD") + + DATE="null" + + DATE=$(add_instance_admin "$INSTANCE_URL" "$PAT" "$HUMAN_USER_ID") + + DATE="null" + DATE=$(delete_auto_service_user "$INSTANCE_URL" "$PAT") + if [ "$DATE" = "null" ]; then + echo "Failed deleting auto service user" + echo "Please remove it manually" + fi + + export NETBIRD_AUTH_CLIENT_ID=$DASHBOARD_APPLICATION_CLIENT_ID + export NETBIRD_AUTH_CLIENT_ID_CLI=$CLI_APPLICATION_CLIENT_ID + export NETBIRD_IDP_MGMT_CLIENT_ID=$SERVICE_USER_CLIENT_ID + export NETBIRD_IDP_MGMT_CLIENT_SECRET=$SERVICE_USER_CLIENT_SECRET + export ZITADEL_ADMIN_USERNAME + export ZITADEL_ADMIN_PASSWORD +} + +initEnvironment() { + CADDY_SECURE_DOMAIN="" + ZITADEL_EXTERNALSECURE="false" + ZITADEL_TLS_MODE="disabled" + ZITADEL_MASTERKEY="$(openssl rand -base64 32 | head -c 32)" + USING_DOMAIN="true" + NETBIRD_PORT=80 + NETBIRD_HTTP_PROTOCOL="http" + TURN_USER="self" + TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g') + TURN_MIN_PORT=49152 + TURN_MAX_PORT=65535 + + NETBIRD_DOMAIN=$NETBIRD_DOMAIN + if [ "$NETBIRD_DOMAIN-x" == "-x" ] ; then + echo "NETBIRD_DOMAIN is not set, using the main IP address" + NETBIRD_DOMAIN=$(get_main_ip_address) + USING_DOMAIN="false" + fi + + if [ "$NETBIRD_DOMAIN" == "localhost" ]; then + USING_DOMAIN="false" + fi + + if [ $USING_DOMAIN == "true" ]; then + ZITADEL_EXTERNALSECURE="true" + ZITADEL_TLS_MODE="external" + NETBIRD_PORT=443 + CADDY_SECURE_DOMAIN=", $NETBIRD_DOMAIN:$NETBIRD_PORT" + NETBIRD_HTTP_PROTOCOL="https" + fi + + if [[ "$OSTYPE" == "darwin"* ]]; then + ZIDATE_TOKEN_EXPIRATION_DATE=$(date -u -v+30M "+%Y-%m-%dT%H:%M:%SZ") + else + ZIDATE_TOKEN_EXPIRATION_DATE=$(date -u -d "+30 minutes" "+%Y-%m-%dT%H:%M:%SZ") + fi + + DOCKER_COMPOSE_COMMAND=$(check_docker_compose) + + if [ -f zitadel.env ]; then + echo "Generated files already exists, if you want to reinitialize the environment, please remove them first." + echo "You can use the following commands:" + echo " $DOCKER_COMPOSE_COMMAND down --volumes # to remove all containers and volumes" + echo " rm -f docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token" + echo "Be aware that this will remove all data from the database and you will have to reconfigure the dashboard." + exit 1 + fi + + echo Rendering initial files... + renderDockerCompose > docker-compose.yml + renderCaddyfile > Caddyfile + renderZitadelEnv > zitadel.env + echo "" > dashboard.env + + mkdir -p machinekey + chmod 777 machinekey + + init_crdb + + echo Starting zidatel + $DOCKER_COMPOSE_COMMAND up -d caddy zitadel crdb + init_zitadel + + echo Rendering NetBird files... + renderTurnServerConf > turnserver.conf + renderManagementJson > management.json + renderDashboardEnv > dashboard.env + + echo Starting NetBird services + $DOCKER_COMPOSE_COMMAND up -d + echo "Done!" + echo "You can access the NetBird dashboard at $NETBIRD_HTTP_PROTOCOL://$NETBIRD_DOMAIN:$NETBIRD_PORT" + echo "Login with the following credentials:" + echo "Username: $ZITADEL_ADMIN_USERNAME" | tee .env + echo "Password: $ZITADEL_ADMIN_PASSWORD" | tee -a .env +} + +renderCaddyfile() { + cat <