diff --git a/.github/scripts/delete-instance.sh b/.github/scripts/delete-instance.sh index 4773852..9b2885f 100755 --- a/.github/scripts/delete-instance.sh +++ b/.github/scripts/delete-instance.sh @@ -7,6 +7,8 @@ # DELETE_HARBOR true to delete the Harbor artifact (default false) # IMAGE_TAG Harbor tag to delete when DELETE_HARBOR=true # REGISTRY_USERNAME, REGISTRY_PASSWORD, REGISTRY_HOST +# PRESERVE_TLS_DIR if set, export TLS secret here before wiping the namespace +# (for GHA artifact restore on redeploy) set -u FAILED=0 @@ -21,6 +23,7 @@ NAMESPACE="ci-${INSTANCE}-ns-pm4" RELEASE="ci-${INSTANCE}" deploy_db="pm4_ci-${INSTANCE}%" deploy_ai="pm4_ci-${INSTANCE}_ai" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ ${#deploy_db} -lt 12 ]; then echo "ERROR: deploy_db safety check failed (length ${#deploy_db})" @@ -29,19 +32,57 @@ fi set +e -if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then - echo "Deleting Instance :: ${RELEASE}" - helm delete "${RELEASE}" || true - kubectl delete namespace "${NAMESPACE}" || true +preserve_tls() { + if [ -z "${PRESERVE_TLS_DIR:-}" ]; then + return 0 + fi + echo "Preserving TLS secret before delete (PRESERVE_TLS_DIR=${PRESERVE_TLS_DIR})" + # shellcheck disable=SC1091 + bash "${SCRIPT_DIR}/export-tls-secret.sh" +} + +delete_k8s() { + if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then + echo "Deleting Instance :: ${RELEASE}" + helm delete "${RELEASE}" || true + kubectl delete namespace "${NAMESPACE}" || true + echo "The instance [https://ci-${INSTANCE}.engk8s.processmaker.net] was removed from the cluster." + return 0 + fi + + echo "Namespace ${NAMESPACE} not found; skipping cluster delete." + return 0 +} + +delete_rds() { + echo "Cleaning RDS for instance ${INSTANCE} (always runs, even when the namespace is already gone)." set -o pipefail - mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -N -e "SHOW DATABASES LIKE '${deploy_db}'" \ - | xargs -r -I{} mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`{}\`;" - if [ $? -ne 0 ]; then - echo "WARN: Failed dropping databases matching ${deploy_db}" + matching_dbs=$( + mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -N -e "SHOW DATABASES LIKE '${deploy_db}'" + ) + list_status=$? + set +o pipefail + + if [ "${list_status}" -ne 0 ]; then + echo "WARN: Failed listing databases matching ${deploy_db}" FAILED=1 + return 1 + fi + + if [ -z "${matching_dbs}" ]; then + echo "No databases matching ${deploy_db} on RDS (already clean)." + else + echo "Dropping databases matching ${deploy_db}:" + echo "${matching_dbs}" + set -o pipefail + echo "${matching_dbs}" | xargs -r -I{} mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`{}\`;" + if [ $? -ne 0 ]; then + echo "WARN: Failed dropping databases matching ${deploy_db}" + FAILED=1 + fi + set +o pipefail fi - set +o pipefail mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`${deploy_ai}\`;" if [ $? -ne 0 ]; then @@ -61,10 +102,12 @@ if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then FAILED=1 fi - echo "The instance [https://ci-${INSTANCE}.engk8s.processmaker.net] was deleted." -else - echo "Namespace ${NAMESPACE} not found; nothing to delete on the cluster." -fi + return 0 +} + +preserve_tls +delete_k8s +delete_rds if [ "${DELETE_HARBOR}" = "true" ]; then if [ -z "${IMAGE_TAG:-}" ]; then diff --git a/.github/scripts/deploy-instance.sh b/.github/scripts/deploy-instance.sh old mode 100644 new mode 100755 index ccf185c..c7024da --- a/.github/scripts/deploy-instance.sh +++ b/.github/scripts/deploy-instance.sh @@ -4,9 +4,35 @@ current_datetime=$(echo -n ${CURRENT_DATE} | md5sum | head -c 10) echo "NAMESPACE : ci-{{INSTANCE}}-ns-pm4" helm repo add processmaker ${HELM_REPO} --username ${HELM_USERNAME} --password ${HELM_PASSWORD} && helm repo update +helm_common_sets() { + echo \ + --set deploy.web.octane.enable=${OCTANE} \ + --set deploy.pmai.openaiApiKey=${OPENAI_API_KEY} \ + --set analytics.awsAccessKey=${ANALYTICS_AWS_ACCESS_KEY} \ + --set analytics.awsSecretKey=${ANALYTICS_AWS_SECRET_KEY} \ + --set dockerRegistry.password=${REGISTRY_PASSWORD} \ + --set dockerRegistry.url=${REGISTRY_HOST} \ + --set dockerRegistry.username=${REGISTRY_USERNAME} \ + --set twilio.sid=${TWILIO_SID} \ + --set twilio.token=${TWILIO_TOKEN} \ + --set appVersion=${APP_VERSION} +} + +# Optional: TLS_SECRET_DIR with a stashed LE cert from a prior delete (GHA artifact). +# Restore before helm install so cert-manager does not burn a new Let's Encrypt issuance. +restore_stashed_tls() { + if [ -z "${TLS_SECRET_DIR:-}" ]; then + return 0 + fi + export INSTANCE="{{INSTANCE}}" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + bash "${SCRIPT_DIR}/restore-tls-secret.sh" +} + if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then echo "New instance. Creating Namespace" kubectl create namespace ci-{{INSTANCE}}-ns-pm4 + restore_stashed_tls echo "Creating DB" # Use admin password from secrets echo "Update instance yamls" @@ -34,20 +60,14 @@ if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then APP_VERSION=$(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g" | sed "s/refs-heads-//g") helm install --timeout 75m -f .github/templates/instance.yaml ci-{{INSTANCE}} processmaker/enterprise \ - --set deploy.web.octane.enable=${OCTANE} \ - --set deploy.pmai.openaiApiKey=${OPENAI_API_KEY} \ - --set analytics.awsAccessKey=${ANALYTICS_AWS_ACCESS_KEY} \ - --set analytics.awsSecretKey=${ANALYTICS_AWS_SECRET_KEY} \ - --set dockerRegistry.password=${REGISTRY_PASSWORD} \ - --set dockerRegistry.url=${REGISTRY_HOST} \ - --set dockerRegistry.username=${REGISTRY_USERNAME} \ - --set twilio.sid=${TWILIO_SID} \ - --set twilio.token=${TWILIO_TOKEN} \ - --set appVersion=${APP_VERSION} \ + $(helm_common_sets) \ --version ${versionHelm} else echo "Instance exists. Running upgrade and bouncing pods" - helm upgrade --timeout 60m ci-{{INSTANCE}} processmaker/enterprise --version ${versionHelm} + APP_VERSION=$(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g" | sed "s/refs-heads-//g") + helm upgrade --timeout 60m -f .github/templates/instance.yaml ci-{{INSTANCE}} processmaker/enterprise \ + $(helm_common_sets) \ + --version ${versionHelm} #Bounce pods webPod=$(kubectl get pods -n ci-{{INSTANCE}}-ns-pm4|grep web|awk '{print $1}') diff --git a/.github/scripts/disable-testbench-relay.sh b/.github/scripts/disable-testbench-relay.sh new file mode 100755 index 0000000..10799f0 --- /dev/null +++ b/.github/scripts/disable-testbench-relay.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +INSTANCE="${1:?INSTANCE id required}" +RELEASE="ci-${INSTANCE}" + +if [ -z "${versionHelm:-}" ]; then + echo "versionHelm is required" + exit 1 +fi + +if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then + echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required" + exit 1 +fi + +helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true +helm repo update + +echo "Disabling testbench runner relay for release ${RELEASE}..." +# --no-hooks: relay-only change; skip post-upgrade update-pm4 (and other hooks). +helm upgrade --timeout 10m "${RELEASE}" processmaker/enterprise \ + --reuse-values \ + --no-hooks \ + --set testbenchRunnerRelay.enable=false \ + --set testbenchRunnerRelay.tailscale.authKey= \ + --version "${versionHelm}" + +echo "Relay disabled." diff --git a/.github/scripts/enable-testbench-relay.sh b/.github/scripts/enable-testbench-relay.sh new file mode 100755 index 0000000..544313e --- /dev/null +++ b/.github/scripts/enable-testbench-relay.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -euo pipefail + +INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}" +RELEASE="ci-${INSTANCE}" +NAMESPACE="${RELEASE}-ns-pm4" +DEPLOYMENT="${RELEASE}-testbench-runner-relay" + +if [ -z "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then + echo "TAILSCALE_AUTH_KEY_CI_RELAY is required" + exit 1 +fi + +RELAY_VALUES="${RELAY_VALUES_FILE:-.github/templates/testbench-relay.yaml}" +if [ ! -f "${RELAY_VALUES}" ]; then + echo "Relay values file not found: ${RELAY_VALUES}" + exit 1 +fi + +RELAY_VALUES_RENDERED=$(mktemp) +sed "s#{{INSTANCE}}#${INSTANCE}#g" "${RELAY_VALUES}" > "${RELAY_VALUES_RENDERED}" +trap 'rm -f "${RELAY_VALUES_RENDERED}"' EXIT + +# Prefer the chart cloned by the Common action (ci:k8s-branch) so unpublished +# template fixes are picked up without waiting on a Harbor chart publish. +CHART_REF="${PM4_K8S_CHART:-}" +if [ -z "${CHART_REF}" ] && [ -f ../pm4-k8s-distribution/charts/enterprise/Chart.yaml ]; then + CHART_REF="../pm4-k8s-distribution/charts/enterprise" +fi +if [ -z "${CHART_REF}" ] && [ -f pm4-k8s-distribution/charts/enterprise/Chart.yaml ]; then + CHART_REF="pm4-k8s-distribution/charts/enterprise" +fi + +HELM_CHART_ARGS=() +if [ -n "${CHART_REF}" ]; then + echo "Using local enterprise chart: ${CHART_REF}" + HELM_CHART_ARGS=("${CHART_REF}") +else + if [ -z "${versionHelm:-}" ]; then + echo "versionHelm is required when no local chart is available" + exit 1 + fi + if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then + echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required when no local chart is available" + exit 1 + fi + helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true + helm repo update + HELM_CHART_ARGS=(processmaker/enterprise --version "${versionHelm}") +fi + +echo "Enabling testbench runner relay for release ${RELEASE} (upstream tb-ci-${INSTANCE})..." +# --no-hooks: instance is already ready; skip post-upgrade update-pm4 (and other hooks). +helm upgrade --timeout 15m "${RELEASE}" "${HELM_CHART_ARGS[@]}" \ + --reuse-values \ + --no-hooks \ + -f "${RELAY_VALUES_RENDERED}" \ + --set testbenchRunnerRelay.enable=true \ + --set "testbenchRunnerRelay.upstreamHost=tb-ci-${INSTANCE}" \ + --set "testbenchRunnerRelay.tailscale.authKey=${TAILSCALE_AUTH_KEY_CI_RELAY}" + +if kubectl get deployment "${DEPLOYMENT}" -n "${NAMESPACE}" >/dev/null 2>&1; then + if ! kubectl rollout status "deployment/${DEPLOYMENT}" -n "${NAMESPACE}" --timeout=180s; then + echo "Relay rollout failed; collecting diagnostics..." + kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true + kubectl describe pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true + kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=200 || true + kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --previous --tail=200 || true + exit 1 + fi +else + echo "Warning: deployment ${DEPLOYMENT} not found after upgrade" + kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true + exit 1 +fi + +echo "Testbench runner relay is ready." diff --git a/.github/scripts/ensure-testbench-relay.sh b/.github/scripts/ensure-testbench-relay.sh new file mode 100755 index 0000000..806dedc --- /dev/null +++ b/.github/scripts/ensure-testbench-relay.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Prepare an existing in-cluster testbench relay for a local/CI runner join (no Helm). +# Run before the runner Tailscale node comes up (start.sh with ENSURE_TESTBENCH_RELAY_SCRIPT). +set -euo pipefail + +INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}" +RELEASE="ci-${INSTANCE}" +NAMESPACE="${RELEASE}-ns-pm4" +DEPLOYMENT="${RELEASE}-testbench-runner-relay" +RELAY_DEPLOY="deploy/${DEPLOYMENT}" +CONFIGMAP="${RELEASE}-testbench-runner-relay" +SECRET="${RELEASE}-testbench-relay-tailscale" +UPSTREAM_HOST="tb-ci-${INSTANCE}" +RELAY_HAPROXY_CHECK_INTER="${RELAY_HAPROXY_CHECK_INTER:-60s}" + +if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl is required to ensure the testbench runner relay" + exit 1 +fi + +if ! kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" >/dev/null 2>&1; then + echo "Relay deployment not found: ${DEPLOYMENT} in namespace ${NAMESPACE}." + echo "" + echo "The testbench runner relay was never enabled for this CI instance." + echo "Run a CI testbench job once (enable-testbench-relay.sh), or use Helm with" + echo "enable-testbench-relay.sh when HELM credentials are available." + exit 1 +fi + +echo "Ensuring testbench runner relay for ${RELEASE} (upstream ${UPSTREAM_HOST})..." + +if kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" >/dev/null 2>&1; then + haproxy_cfg="$(kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" -o jsonpath='{.data.haproxy\.cfg}')" + patched_cfg="$(printf '%s' "${haproxy_cfg}" \ + | sed -E "s/server runner [^:]+:/server runner ${UPSTREAM_HOST}:/g" \ + | sed -E "s/check inter [0-9]+[smhd]+/check inter ${RELAY_HAPROXY_CHECK_INTER}/g")" + if ! printf '%s' "${patched_cfg}" | grep -q 'hard-stop-after'; then + patched_cfg="$(printf '%s' "${patched_cfg}" | awk ' + /^[[:space:]]*maxconn / { print; print " hard-stop-after 5s"; next } + { print } + ')" + fi + if [ "${patched_cfg}" = "${haproxy_cfg}" ]; then + echo "ConfigMap upstream (${UPSTREAM_HOST}) and health check interval (${RELAY_HAPROXY_CHECK_INTER}) already set." + else + echo "Patching ConfigMap ${CONFIGMAP} (upstream ${UPSTREAM_HOST}, check inter ${RELAY_HAPROXY_CHECK_INTER})..." + kubectl create configmap "${CONFIGMAP}" -n "${NAMESPACE}" \ + --from-literal=haproxy.cfg="${patched_cfg}" \ + --dry-run=client -o yaml \ + | kubectl apply -f - + fi +else + echo "WARNING: ConfigMap ${CONFIGMAP} not found; skipping HAProxy patch" +fi + +if [ -n "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then + if kubectl get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then + echo "Refreshing relay Tailscale auth key secret ${SECRET}..." + kubectl create secret generic "${SECRET}" -n "${NAMESPACE}" \ + --from-literal=TS_AUTHKEY="${TAILSCALE_AUTH_KEY_CI_RELAY}" \ + --dry-run=client -o yaml \ + | kubectl apply -f - + else + echo "WARNING: Secret ${SECRET} not found; skipping auth key refresh" + fi +fi + +echo "Applying fast relay rollout settings (Recreate strategy, exec probes, grace period)..." +# JSON patch (not --type=merge): merge patch replaces the entire containers[] +# array and drops required fields like image. +# Replace whole probes so a prior tcpSocket handler cannot remain beside exec. +TS_PROBE='tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1' +grace_op="add" +if kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" \ + -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}' 2>/dev/null | grep -q '[0-9]'; then + grace_op="replace" +fi +kubectl patch "${RELAY_DEPLOY}" -n "${NAMESPACE}" --type=json -p "$(cat </dev/null 2>&1; then + echo "Namespace ${NAMESPACE} not found; nothing to stash" + exit 0 +fi + +if ! kubectl get "secret/${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then + echo "Secret ${SECRET} not found in ${NAMESPACE}; nothing to stash" + exit 0 +fi + +if ! kubectl get "secret/${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.crt}' 2>/dev/null | base64 -d >"${PRESERVE_TLS_DIR}/tls.crt"; then + echo "WARN: Failed reading tls.crt from ${SECRET}" + rm -rf "${PRESERVE_TLS_DIR}" + mkdir -p "${PRESERVE_TLS_DIR}" + exit 0 +fi + +if ! kubectl get "secret/${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.key}' 2>/dev/null | base64 -d >"${PRESERVE_TLS_DIR}/tls.key"; then + echo "WARN: Failed reading tls.key from ${SECRET}" + rm -rf "${PRESERVE_TLS_DIR}" + mkdir -p "${PRESERVE_TLS_DIR}" + exit 0 +fi + +# Optional CA bundle (present for some issuers; harmless if missing). +kubectl get "secret/${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.ca\.crt}' 2>/dev/null | base64 -d >"${PRESERVE_TLS_DIR}/ca.crt" || true +if [ ! -s "${PRESERVE_TLS_DIR}/ca.crt" ]; then + rm -f "${PRESERVE_TLS_DIR}/ca.crt" +fi + +if [ ! -s "${PRESERVE_TLS_DIR}/tls.crt" ] || [ ! -s "${PRESERVE_TLS_DIR}/tls.key" ]; then + echo "WARN: Exported tls.crt/tls.key empty; not stashing" + rm -rf "${PRESERVE_TLS_DIR}" + mkdir -p "${PRESERVE_TLS_DIR}" + exit 0 +fi + +if command -v openssl >/dev/null 2>&1; then + if ! openssl x509 -in "${PRESERVE_TLS_DIR}/tls.crt" -noout -checkend 86400 >/dev/null 2>&1; then + echo "WARN: Certificate expired or expires within 24h; not stashing" + rm -rf "${PRESERVE_TLS_DIR}" + mkdir -p "${PRESERVE_TLS_DIR}" + exit 0 + fi + echo "Stashing TLS cert:" + openssl x509 -in "${PRESERVE_TLS_DIR}/tls.crt" -noout -subject -issuer -dates 2>/dev/null || true +else + echo "openssl not available; skipping expiry check" +fi + +chmod 600 "${PRESERVE_TLS_DIR}/tls.key" +# Non-dot marker so actions/upload-artifact reliably includes it. +touch "${PRESERVE_TLS_DIR}/stashed" +echo "TLS secret ${SECRET} exported to ${PRESERVE_TLS_DIR}" +exit 0 diff --git a/.github/scripts/restore-tls-secret.sh b/.github/scripts/restore-tls-secret.sh new file mode 100755 index 0000000..d72f7ab --- /dev/null +++ b/.github/scripts/restore-tls-secret.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Restore a stashed CI TLS secret into the instance namespace before helm install +# so cert-manager does not request a new Let's Encrypt certificate. +# Required env: +# INSTANCE instance id +# TLS_SECRET_DIR directory with tls.crt / tls.key from export-tls-secret.sh +# Creates the namespace if missing. Exits 0 if there is nothing to restore. +set -u + +INSTANCE="${INSTANCE:-}" +TLS_SECRET_DIR="${TLS_SECRET_DIR:-}" +NAMESPACE="ci-${INSTANCE}-ns-pm4" +SECRET="ci-${INSTANCE}-tls-secret" + +if [ -z "${INSTANCE}" ] || [ -z "${TLS_SECRET_DIR}" ]; then + echo "TLS restore skipped (INSTANCE or TLS_SECRET_DIR unset)" + exit 0 +fi + +if [ ! -s "${TLS_SECRET_DIR}/tls.crt" ] || [ ! -s "${TLS_SECRET_DIR}/tls.key" ]; then + echo "No stashed TLS secret at ${TLS_SECRET_DIR}; cert-manager will issue if needed" + exit 0 +fi + +if command -v openssl >/dev/null 2>&1; then + if ! openssl x509 -in "${TLS_SECRET_DIR}/tls.crt" -noout -checkend 86400 >/dev/null 2>&1; then + echo "WARN: Stashed certificate expired or expires within 24h; not restoring" + exit 0 + fi +fi + +if ! kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then + echo "Creating namespace ${NAMESPACE} for TLS restore" + kubectl create namespace "${NAMESPACE}" +fi + +echo "Restoring TLS secret ${SECRET} into ${NAMESPACE}" +kubectl create secret tls "${SECRET}" \ + --cert="${TLS_SECRET_DIR}/tls.crt" \ + --key="${TLS_SECRET_DIR}/tls.key" \ + -n "${NAMESPACE}" \ + --dry-run=client -o yaml | kubectl apply -f - + +if [ -s "${TLS_SECRET_DIR}/ca.crt" ]; then + # Attach ca.crt when present (create secret tls does not include it). + CA_B64=$(base64 <"${TLS_SECRET_DIR}/ca.crt" | tr -d '\n') + kubectl patch secret "${SECRET}" -n "${NAMESPACE}" --type merge -p "{\"data\":{\"ca.crt\":\"${CA_B64}\"}}" +fi + +# Help cert-manager adopt the existing secret for the ingress-managed Certificate. +kubectl annotate secret "${SECRET}" -n "${NAMESPACE}" \ + "cert-manager.io/certificate-name=${SECRET}" \ + --overwrite >/dev/null + +echo "TLS secret restored; cert-manager should adopt instead of re-issuing" +exit 0 diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh new file mode 100755 index 0000000..ca9266d --- /dev/null +++ b/.github/scripts/verify-testbench-relay.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# Fail-fast: prove PM → in-cluster relay → Tailscale → runner mail ports. +# Run after the runner has joined the tailnet as tb-ci- (start.sh). +# +# Restarts the relay Deployment first so HAProxy/MagicDNS pick up the current +# ephemeral runner IP (same hostname tb-ci-* can point at a dead peer from a +# previous testbench run if the relay pod was left running). +set -euo pipefail + +INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}" +RELEASE="ci-${INSTANCE}" +NAMESPACE="${RELEASE}-ns-pm4" +RELAY_DEPLOY="deploy/${RELEASE}-testbench-runner-relay" +WEB_DEPLOY="deploy/${RELEASE}-processmaker-web" +UPSTREAM_HOST="tb-ci-${INSTANCE}" +RELAY_SVC="${RELEASE}-testbench-relay" +NC_TIMEOUT_SEC="${RELAY_NC_TIMEOUT_SEC:-5}" +# Allow MagicDNS / DERP / HAProxy health to settle after runner joins / relay restart. +WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-90}" +POLL_SEC="${RELAY_VERIFY_POLL_SEC:-10}" +ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-90}" + +if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl is required to verify the testbench runner relay" + exit 1 +fi + +dump_diagnostics() { + echo "" + echo "======== testbench runner relay diagnostics ========" + echo "--- pods ---" + kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true + echo "" + echo "--- relay deploy / svc ---" + kubectl get deploy,svc -n "${NAMESPACE}" -l service=testbench-runner-relay 2>/dev/null || \ + kubectl get deploy "${RELEASE}-testbench-runner-relay" -n "${NAMESPACE}" 2>/dev/null || true + kubectl get svc "${RELAY_SVC}" -n "${NAMESPACE}" 2>/dev/null || true + echo "" + echo "--- relay: tailscale status ---" + kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- tailscale status 2>&1 || true + echo "" + echo "--- relay: peer ${UPSTREAM_HOST} (json) ---" + kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- tailscale status --json 2>&1 \ + | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) +except Exception as e: + print('failed to parse status json:', e) + sys.exit(0) +self = d.get('Self') or {} +print('Self:', self.get('HostName'), 'Tags:', self.get('Tags'), 'Online:', self.get('Online')) +for p in (d.get('Peer') or {}).values(): + host = p.get('HostName') or '' + if host == '${UPSTREAM_HOST}' or '${INSTANCE}' in host: + print( + 'Peer:', host, + 'Tags:', p.get('Tags'), + 'Online:', p.get('Online'), + 'RxBytes:', p.get('RxBytes'), + 'TxBytes:', p.get('TxBytes'), + 'LastSeen:', p.get('LastSeen'), + 'Relay:', p.get('Relay'), + 'CurAddr:', p.get('CurAddr'), + ) +" 2>&1 || true + echo "" + echo "--- relay logs (tail 100) ---" + kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=100 2>&1 || true + echo "" + echo "--- previous relay logs (if restarted) ---" + kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --previous --tail=50 2>&1 || true + echo "" + echo "Hints:" + echo " - Confirm Tailscale ACL allows tag:ci-relay -> tag:ci-runner:587 and :993" + echo " - Confirm runner auth key has tag:ci-runner and relay key has tag:ci-relay" + echo " - Confirm runner hostname is ${UPSTREAM_HOST} (not ${UPSTREAM_HOST}-1); use TAILSCALE_API_KEY_CI cleanup" + echo " - HAProxy 'no server available' / stale IP: relay must restart after runner joins" + echo " - Exit 137 on relay often means OOM; check memory limits" + echo "====================================================" +} + +echo "Verifying testbench runner relay path for ${RELEASE} (upstream ${UPSTREAM_HOST})..." + +if ! kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" >/dev/null 2>&1; then + echo "Relay deployment not found: ${RELAY_DEPLOY} in ${NAMESPACE}" + dump_diagnostics + exit 1 +fi + +echo "--- restart relay (refresh HAProxy upstream for ${UPSTREAM_HOST}) ---" +kubectl rollout restart "${RELAY_DEPLOY}" -n "${NAMESPACE}" +if ! kubectl rollout status "${RELAY_DEPLOY}" -n "${NAMESPACE}" --timeout="${ROLLOUT_TIMEOUT_SEC}s"; then + echo "FAIL: relay rollout after restart did not complete" + dump_diagnostics + exit 1 +fi + +# HAProxy may still be binding after the pod becomes Ready (esp. while Tailscale +# finishes login). Retry cluster-local probes with the same settle window. +deadline=$((SECONDS + WAIT_SEC)) +attempt=0 +web_ok=false +while true; do + attempt=$((attempt + 1)) + echo "--- web -> ${RELAY_SVC}:587 (attempt ${attempt}, ${NC_TIMEOUT_SEC}s) ---" + if kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \ + sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 587"; then + echo "--- web -> ${RELAY_SVC}:993 ---" + if kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \ + sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 993"; then + web_ok=true + break + fi + echo "SMTP OK but IMAP :993 failed; will retry if time remains..." + else + echo "web -> relay :587 not reachable yet..." + fi + + if [ "${SECONDS}" -ge "${deadline}" ]; then + break + fi + sleep "${POLL_SEC}" +done + +if [ "${web_ok}" != "true" ]; then + echo "FAIL: web pod cannot reach relay Service :587/:993 within ${WAIT_SEC}s" + dump_diagnostics + exit 1 +fi + +deadline=$((SECONDS + WAIT_SEC)) +attempt=0 +while true; do + attempt=$((attempt + 1)) + echo "--- relay -> ${UPSTREAM_HOST}:587 (attempt ${attempt}, ${NC_TIMEOUT_SEC}s) ---" + if kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- \ + sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${UPSTREAM_HOST} 587"; then + echo "--- relay -> ${UPSTREAM_HOST}:993 ---" + if kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- \ + sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${UPSTREAM_HOST} 993"; then + echo "OK: cluster relay path is reachable (web -> relay svc -> ${UPSTREAM_HOST}:587/993)." + exit 0 + fi + echo "SMTP OK but IMAP :993 failed; will retry if time remains..." + else + echo "SMTP :587 not reachable yet..." + fi + + if [ "${SECONDS}" -ge "${deadline}" ]; then + break + fi + sleep "${POLL_SEC}" +done + +echo "FAIL: relay pod cannot reach ${UPSTREAM_HOST}:587/993 over Tailscale within ${WAIT_SEC}s" +dump_diagnostics +exit 1 diff --git a/.github/templates/testbench-relay.yaml b/.github/templates/testbench-relay.yaml new file mode 100644 index 0000000..e7a5fc9 --- /dev/null +++ b/.github/templates/testbench-relay.yaml @@ -0,0 +1,14 @@ +# Overlay for testbench jobs only (helm upgrade --reuse-values -f this file). +# Do not enable relay on default CI deploy. +testbenchRunnerRelay: + enable: true + upstreamHost: tb-ci-{{INSTANCE}} + forwards: + - name: smtp + listen: 587 + upstream: 587 + protocol: tcp + - name: imaps + listen: 993 + upstream: 993 + protocol: tcp diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index c8d9655..e700562 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -78,7 +78,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: processmaker/.github + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} - name: Common uses: ./.github/actions/common @@ -177,8 +179,9 @@ jobs: - name: Checkout .github repo uses: actions/checkout@v4 with: - repository: processmaker/.github - ref: main + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} - name: Common uses: ./.github/actions/common @@ -207,7 +210,22 @@ jobs: - name: Authenticate with Amazon EKS run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng - + + - name: Resolve instance id + id: instance + run: | + instance=$(echo -n "${{ env.IMAGE_TAG }}" | md5sum | head -c 10) + echo "instance=${instance}" >> "$GITHUB_OUTPUT" + echo "INSTANCE: ${instance}" + + # Restored before helm install so cert-manager does not re-hit Let's Encrypt. + # Missing artifact is normal when deleteEKS was skipped or had no usable cert. + - name: Download stashed TLS secret + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ci-tls-${{ steps.instance.outputs.instance }} + path: ${{ runner.temp }}/ci-tls - name: Deploy instance EKS env: @@ -235,8 +253,9 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} OCTANE: ${{ env.OCTANE }} + TLS_SECRET_DIR: ${{ runner.temp }}/ci-tls run: | - instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + instance="${{ steps.instance.outputs.instance }}" echo "INSTANCE: $instance" echo "IMAGE_TAG: $IMAGE_TAG" @@ -260,7 +279,7 @@ jobs: cat .github/templates/instance.yaml echo "=== Checking db.yaml after replacements ===" cat .github/templates/db.yaml - chmod +x .github/scripts/deploy-instance.sh + chmod +x .github/scripts/deploy-instance.sh .github/scripts/restore-tls-secret.sh bash .github/scripts/deploy-instance.sh fi if [ "$MULTITENANCY" = "true" ]; then @@ -283,7 +302,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: processmaker/.github + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} - name: Common uses: ./.github/actions/common @@ -364,22 +385,101 @@ jobs: contains(github.event.pull_request.body, 'ci:run-testbench') runs-on: ${{ vars.RUNNER }} steps: + - name: Resolve companion branch + id: companion + shell: bash + env: + GH_TOKEN: ${{ secrets.GIT_TOKEN }} + run: | + # Nested reusable workflows leave github.job_workflow_ref empty; job.workflow_sha + # is reliable. Map that SHA to a branch tip so testbench tracks the same series + # (e.g. the calling feature branch). Fall back to ci:k8s-branch, then main. + sha='${{ job.workflow_sha }}' + repo='${{ job.workflow_repository }}' + echo "workflow_sha=$sha repo=$repo" + ref="$(gh api "repos/${repo}/commits/${sha}/branches-where-head" --jq '.[0].name // empty' || true)" + if [ -z "$ref" ] && [[ "${CI_PR_BODY:-}" =~ ci:k8s-branch:([^[:space:]]+) ]]; then + ref="${BASH_REMATCH[1]}" + echo "branches-where-head empty; using ci:k8s-branch=$ref" + fi + if [ -z "$ref" ]; then + ref=main + fi + echo "companion_ref=$ref" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + + - name: Checkout dot-github + uses: actions/checkout@v4 + with: + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: dot-github + token: ${{ secrets.GIT_TOKEN }} + + - name: Common + uses: ./dot-github/.github/actions/common + with: + token: ${{ secrets.GIT_TOKEN }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Authenticate with Amazon EKS + run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng + - name: Checkout testbench uses: actions/checkout@v4 with: repository: ProcessMaker/testbench + ref: ${{ steps.companion.outputs.ref }} path: testbench token: ${{ secrets.GIT_TOKEN }} + # Must run before enable-testbench-relay so new relay/runner reclaim stable hostnames. + - name: Cleanup stale Tailscale CI nodes + working-directory: testbench + env: + TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }} + run: | + instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + if [ -z "${TAILSCALE_API_KEY_CI}" ]; then + echo "ERROR: org secret TAILSCALE_API_KEY_CI is required (Tailscale API access token tskey-api-…)" + exit 1 + fi + npm ci + npx tsx src/index.ts cleanup-tailscale --instance "$instance" --targets all + + - name: Enable testbench runner relay + working-directory: dot-github + env: + HELM_REPO: ${{ secrets.HELM_REPO }} + HELM_USERNAME: ${{ secrets.HELM_USERNAME }} + HELM_PASSWORD: ${{ secrets.HELM_PASSWORD }} + TAILSCALE_AUTH_KEY_CI_RELAY: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RELAY }} + run: | + instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + chmod +x .github/scripts/enable-testbench-relay.sh + .github/scripts/enable-testbench-relay.sh "$instance" + - name: Run testbench working-directory: testbench env: SITE_NAME: CI INSTANCE_PASSWORD: ${{ secrets.CI_INSTANCE_PASSWORD }} CONTEXT_PATH: ./context - TCP_TUNNELS: "mailserver:587 mailserver:993" + USE_RUNNER_RELAY: "true" + TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }} + TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }} + ENSURE_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/ensure-testbench-relay.sh + VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh run: | instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + export INSTANCE="$instance" echo "INSTANCE: $instance" if [ "${{ env.MULTITENANCY }}" = "true" ]; then export URL="https://tenant-1.ci-$instance.engk8s.processmaker.net" @@ -394,7 +494,7 @@ jobs: deleteEKS: name: Delete Instance if: github.event.action == 'closed' || inputs.delete != '' || contains(github.event.pull_request.body, 'ci:redeploy') - runs-on: self-hosted + runs-on: [self-hosted, pm-delete-runner] # GITHUB_TOKEN needs write to strip ci:redeploy from the PR body. # Using github.token (not GIT_TOKEN) so the edit does not trigger a new workflow run. permissions: @@ -430,96 +530,63 @@ jobs: username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Checkout .github repo + uses: actions/checkout@v4 + with: + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + - name: Delete instance EKS - run: | + id: delete_instance + env: + USER_MYSQL_ENG: ${{ secrets.USER_MYSQL_ENG }} + PASS_MYSQL_ENG: ${{ secrets.PASS_MYSQL_ENG }} + RDS_ENG: ${{ secrets.RDS_ENG }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + REGISTRY_HOST: ${{ secrets.REGISTRY_HOST }} + run: | set +e - FAILED=0 - - # If inputs.delete does not equal '', set the IMAGE_TAG to the value of inputs.delete if [ "${{ inputs.delete }}" != "" ]; then IMAGE_TAG=${{ inputs.delete }} else IMAGE_TAG=${{ env.IMAGE_TAG }} fi - INSTANCE=$(echo -n $IMAGE_TAG | md5sum | head -c 10) + export INSTANCE=$(echo -n "$IMAGE_TAG" | md5sum | head -c 10) + export IMAGE_TAG + echo "instance=${INSTANCE}" >> "$GITHUB_OUTPUT" - # Delete Harbor image only on PR close / manual delete — not on ci:redeploy - DELETE_HARBOR=false if [ "${{ github.event.action }}" = "closed" ] || [ "${{ inputs.delete }}" != "" ]; then - DELETE_HARBOR=true + export DELETE_HARBOR=true + else + export DELETE_HARBOR=false fi - if kubectl get namespace/ci-$INSTANCE-ns-pm4 ; then - echo "Deleting Instance :: ci-$INSTANCE" - helm delete ci-$INSTANCE || true - kubectl delete namespace ci-$INSTANCE-ns-pm4 || true - - #Drop database - deploy_db="pm4_ci-${INSTANCE}%" - deploy_ai="\`pm4_ci-$INSTANCE_ai\`" - - # check that that string length of $deploy_db is 12 or more as a safety check. If its less than 12, exit now - if [ ${#deploy_db} -lt 12 ]; then - echo "ERROR: deploy_db safety check failed (length ${#deploy_db})" - exit 1 - fi - - # Drop the main database including any tenant databases. - # Continue on failure so a half-built instance still gets cleaned up. - set -o pipefail - mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -h ${{ secrets.RDS_ENG }} -N -e "SHOW DATABASES LIKE '${deploy_db}'" | xargs -r -I{} mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -h ${{ secrets.RDS_ENG }} -e "DROP DATABASE IF EXISTS \`{}\`;" - if [ $? -ne 0 ]; then - echo "WARN: Failed dropping databases matching ${deploy_db}" - FAILED=1 - fi - set +o pipefail - - mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -e "DROP DATABASE IF EXISTS $deploy_ai" -h ${{ secrets.RDS_ENG }} - if [ $? -ne 0 ]; then - echo "WARN: Failed dropping AI database $deploy_ai" - FAILED=1 - fi - - mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -e "DROP USER IF EXISTS 'user_ci-$INSTANCE'@'%'" -h ${{ secrets.RDS_ENG }} - if [ $? -ne 0 ]; then - echo "WARN: Failed dropping user user_ci-$INSTANCE" - FAILED=1 - fi - - mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -e "DROP USER IF EXISTS 'user_ci-$INSTANCE_ai'@'%'" -h ${{ secrets.RDS_ENG }} - if [ $? -ne 0 ]; then - echo "WARN: Failed dropping user user_ci-$INSTANCE_ai" - FAILED=1 - fi - - echo "The instance [https://ci-$INSTANCE.engk8s.processmaker.net] was deleted!!" - else - echo "The pull request does not have an instance on K8s [https://ci-$INSTANCE.engk8s.processmaker.net] not found!!" + # On ci:redeploy, stash the LE TLS secret as a GHA artifact so deploy can + # restore it and avoid burning another Let's Encrypt issuance. + if [ "${{ github.event.action }}" != "closed" ] && [ "${{ inputs.delete }}" = "" ] && [[ "${{ github.event.pull_request.body }}" == *"ci:redeploy"* ]]; then + export PRESERVE_TLS_DIR="${RUNNER_TEMP}/ci-tls" fi - if [ "$DELETE_HARBOR" = "true" ]; then - echo "Deleting image from Harbor: ${IMAGE_TAG}" - HTTP_CODE=$(curl -s -o /tmp/harbor_delete_out -w "%{http_code}" -X DELETE \ - -u ${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }} \ - "https://${{ secrets.REGISTRY_HOST }}/api/v2.0/projects/processmaker/repositories/enterprise/artifacts/${IMAGE_TAG}") - if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then - echo "Harbor image deleted (HTTP $HTTP_CODE)" - elif [ "$HTTP_CODE" = "404" ]; then - echo "Harbor image not found (already deleted)" - else - echo "WARN: Failed deleting Harbor image ${IMAGE_TAG} (HTTP $HTTP_CODE)" - cat /tmp/harbor_delete_out || true - FAILED=1 - fi + echo "INSTANCE=${INSTANCE}" + chmod +x .github/scripts/delete-instance.sh .github/scripts/export-tls-secret.sh + bash .github/scripts/delete-instance.sh + + if [ -n "${PRESERVE_TLS_DIR:-}" ] && [ -f "${PRESERVE_TLS_DIR}/stashed" ]; then + echo "tls_stashed=true" >> "$GITHUB_OUTPUT" else - echo "Skipping Harbor image delete (ci:redeploy)" + echo "tls_stashed=false" >> "$GITHUB_OUTPUT" fi - if [ "$FAILED" -ne 0 ]; then - echo "One or more delete operations failed" - exit 1 - fi - echo "Delete completed successfully" + - name: Upload stashed TLS secret + if: steps.delete_instance.outputs.tls_stashed == 'true' + uses: actions/upload-artifact@v4 + with: + name: ci-tls-${{ steps.delete_instance.outputs.instance }} + path: ${{ runner.temp }}/ci-tls/ + retention-days: 1 + if-no-files-found: warn # Strip ci:redeploy from the PR body so it does not re-trigger on later edits. # Uses github.token (default) so this edit does not start a new workflow run. @@ -553,7 +620,8 @@ jobs: !cancelled() && needs.imageEKS.result == 'success' && github.event.action != 'closed' && - inputs.delete == '' + inputs.delete == '' && + !contains(github.event.pull_request.body, 'ci:skip-phpunit') needs: imageEKS runs-on: ${{ vars.RUNNER }} timeout-minutes: 150 @@ -564,7 +632,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: processmaker/.github + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} - name: Common uses: ./.github/actions/common diff --git a/.github/workflows/js-package.yml b/.github/workflows/js-package.yml index 28e9f6d..b134e9e 100644 --- a/.github/workflows/js-package.yml +++ b/.github/workflows/js-package.yml @@ -54,8 +54,9 @@ jobs: - name: Checkout .github uses: actions/checkout@v4 with: - repository: ProcessMaker/.github - ref: main + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml index 427f832..a12c6b8 100644 --- a/.github/workflows/nightly-testbench-arch.yml +++ b/.github/workflows/nightly-testbench-arch.yml @@ -71,16 +71,32 @@ jobs: kubectl cluster-info --request-timeout=15s - name: Delete instance EKS + id: delete_instance env: INSTANCE: ${{ env.INSTANCE }} USER_MYSQL_ENG: ${{ secrets.USER_MYSQL_ENG }} PASS_MYSQL_ENG: ${{ secrets.PASS_MYSQL_ENG }} RDS_ENG: ${{ secrets.RDS_ENG }} DELETE_HARBOR: "false" + PRESERVE_TLS_DIR: ${{ runner.temp }}/ci-tls run: | echo "INSTANCE: $INSTANCE" - chmod +x .github/scripts/delete-instance.sh + chmod +x .github/scripts/delete-instance.sh .github/scripts/export-tls-secret.sh bash .github/scripts/delete-instance.sh + if [ -f "${PRESERVE_TLS_DIR}/stashed" ]; then + echo "tls_stashed=true" >> "$GITHUB_OUTPUT" + else + echo "tls_stashed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Upload stashed TLS secret + if: steps.delete_instance.outputs.tls_stashed == 'true' + uses: actions/upload-artifact@v4 + with: + name: ci-tls-${{ env.INSTANCE }} + path: ${{ runner.temp }}/ci-tls/ + retention-days: 1 + if-no-files-found: warn deploy: name: deploy-${{ inputs.arch_id }} @@ -132,6 +148,13 @@ jobs: - name: Authenticate with Amazon EKS run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng + - name: Download stashed TLS secret + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ci-tls-${{ env.INSTANCE }} + path: ${{ runner.temp }}/ci-tls + - name: Deploy instance EKS env: IMAGE_TAG: ${{ inputs.image_tag }} @@ -160,6 +183,7 @@ jobs: MULTITENANCY: ${{ env.MULTITENANCY }} CI_PROJECT: processmaker CI_PACKAGE_BRANCH: develop + TLS_SECRET_DIR: ${{ runner.temp }}/ci-tls run: | instance="${{ steps.instance.outputs.instance }}" echo "INSTANCE: $instance" @@ -181,7 +205,7 @@ jobs: sed -i "s#{{PMAI_HOST_WS}}#$PMAI_HOST_WS#g" .github/templates/instance.yaml sed -i "s#{{MYSQL_USERNAME}}#$RDS_ADMIN_USERNAME#g" .github/templates/db.yaml sed -i "s#{{MYSQL_PASSWORD}}#$RDS_ADMIN_PASSWORD#g" .github/templates/db.yaml - chmod +x .github/scripts/deploy-instance.sh + chmod +x .github/scripts/deploy-instance.sh .github/scripts/restore-tls-secret.sh bash .github/scripts/deploy-instance.sh testbench: @@ -195,22 +219,97 @@ jobs: matrix: tenant: ${{ fromJSON(inputs.tenants_json) }} steps: + - name: Resolve companion branch + id: companion + shell: bash + env: + GH_TOKEN: ${{ secrets.GIT_TOKEN }} + run: | + # Nested reusable workflows leave github.job_workflow_ref empty; job.workflow_sha + # is reliable. Map that SHA to a branch tip so testbench tracks the same series + # (e.g. the calling feature branch). Fall back to main. + sha='${{ job.workflow_sha }}' + repo='${{ job.workflow_repository }}' + echo "workflow_sha=$sha repo=$repo" + ref="$(gh api "repos/${repo}/commits/${sha}/branches-where-head" --jq '.[0].name // empty' || true)" + if [ -z "$ref" ]; then + ref=main + fi + echo "companion_ref=$ref" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + + - name: Checkout dot-github + uses: actions/checkout@v4 + with: + # Match the reusable-workflow ref (e.g. the calling feature branch), not main. + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: dot-github + token: ${{ secrets.GIT_TOKEN }} + + - name: Common + uses: ./dot-github/.github/actions/common + with: + token: ${{ secrets.GIT_TOKEN }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Authenticate with Amazon EKS + run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng + - name: Checkout testbench uses: actions/checkout@v4 with: repository: ProcessMaker/testbench + ref: ${{ steps.companion.outputs.ref }} path: testbench token: ${{ secrets.GIT_TOKEN }} + # Must run before enable-testbench-relay so new relay/runner reclaim stable hostnames. + - name: Cleanup stale Tailscale CI nodes + working-directory: testbench + env: + TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }} + run: | + instance="${{ needs.deploy.outputs.instance }}" + if [ -z "${TAILSCALE_API_KEY_CI}" ]; then + echo "ERROR: org secret TAILSCALE_API_KEY_CI is required (Tailscale API access token tskey-api-…)" + exit 1 + fi + npm ci + npx tsx src/index.ts cleanup-tailscale --instance "$instance" --targets all + + - name: Enable testbench runner relay + working-directory: dot-github + env: + HELM_REPO: ${{ secrets.HELM_REPO }} + HELM_USERNAME: ${{ secrets.HELM_USERNAME }} + HELM_PASSWORD: ${{ secrets.HELM_PASSWORD }} + TAILSCALE_AUTH_KEY_CI_RELAY: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RELAY }} + run: | + instance="${{ needs.deploy.outputs.instance }}" + chmod +x .github/scripts/enable-testbench-relay.sh + .github/scripts/enable-testbench-relay.sh "$instance" + - name: Run testbench working-directory: testbench env: SITE_NAME: CI INSTANCE_PASSWORD: ${{ secrets.CI_INSTANCE_PASSWORD }} CONTEXT_PATH: ./context - TCP_TUNNELS: "mailserver:587 mailserver:993" + USE_RUNNER_RELAY: "true" + TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }} + TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }} + ENSURE_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/ensure-testbench-relay.sh + VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh run: | instance="${{ needs.deploy.outputs.instance }}" + export INSTANCE="$instance" tenant="${{ matrix.tenant }}" echo "INSTANCE: $instance" echo "TENANT: $tenant"