diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1cef46ae0c..5c0326e4af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -389,6 +389,7 @@ jobs: MAVEN_USER: ${{ secrets.MAVEN_USER }} MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }} + DOCKER_BASE_CHANGED: ${{ needs.files-changed.outputs.docker-base }} - name: Upload Cucumber Report if: always() @@ -399,6 +400,15 @@ jobs: retention-days: 7 if-no-files-found: warn + - name: Upload Test Reports + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: docker-compose-test-reports + path: testing/reports/ + retention-days: 7 + if-no-files-found: warn + - name: Cucumber Test Report if: always() uses: dorny/test-reporter@b082adf0eced0765477756c2a610396589b8c637 # v2.5.0 diff --git a/build.gradle b/build.gradle index 3a7a529204..b41c54e3b8 100644 --- a/build.gradle +++ b/build.gradle @@ -177,8 +177,6 @@ subprojects { exclude group: 'org.bouncycastle', module: 'bcpkix-jdk15on' exclude group: 'org.bouncycastle', module: 'bcutil-jdk15on' exclude group: 'org.bouncycastle', module: 'bcmail-jdk15on' - // Force a compatible Guava version for spotless - resolutionStrategy.force 'com.google.guava:guava:33.0.0-jre' // Security CVE fixes - hardcoded resolution strategy to ensure safe versions // Primary fixes via explicit dependencies in app/core/build.gradle: diff --git a/docker/embedded/compose/docker-compose-latest-fat-endpoints-disabled.yml b/docker/embedded/compose/docker-compose-latest-fat-endpoints-disabled.yml index dd5048b004..512ba4f144 100644 --- a/docker/embedded/compose/docker-compose-latest-fat-endpoints-disabled.yml +++ b/docker/embedded/compose/docker-compose-latest-fat-endpoints-disabled.yml @@ -2,9 +2,6 @@ services: stirling-pdf: container_name: Stirling-PDF-Fat-Disable-Endpoints - build: - context: ../../.. - dockerfile: docker/embedded/Dockerfile.fat image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:fat build: context: ../../../ diff --git a/docker/embedded/compose/docker-compose-latest-fat-security.yml b/docker/embedded/compose/docker-compose-latest-fat-security.yml index 167c7172e0..9ca92ce60c 100644 --- a/docker/embedded/compose/docker-compose-latest-fat-security.yml +++ b/docker/embedded/compose/docker-compose-latest-fat-security.yml @@ -1,13 +1,10 @@ services: stirling-pdf: container_name: Stirling-PDF-Security-Fat - build: - context: ../../.. - dockerfile: docker/embedded/Dockerfile.fat image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:fat build: context: ../../../ - dockerfile: docker/embedded/Dockerfile + dockerfile: docker/embedded/Dockerfile.fat deploy: resources: limits: diff --git a/docker/embedded/compose/docker-compose-latest-ultra-lite.yml b/docker/embedded/compose/docker-compose-latest-ultra-lite.yml index 1f95f50e38..a5cae8dc54 100644 --- a/docker/embedded/compose/docker-compose-latest-ultra-lite.yml +++ b/docker/embedded/compose/docker-compose-latest-ultra-lite.yml @@ -1,9 +1,6 @@ services: stirling-pdf: container_name: Stirling-PDF-Ultra-Lite - build: - context: ../../.. - dockerfile: docker/embedded/Dockerfile.ultra-lite image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:ultra-lite build: context: ../../../ diff --git a/docker/embedded/compose/test_cicd.yml b/docker/embedded/compose/test_cicd.yml index 55a06a8f7a..d1e08013c2 100644 --- a/docker/embedded/compose/test_cicd.yml +++ b/docker/embedded/compose/test_cicd.yml @@ -1,9 +1,6 @@ services: stirling-pdf: container_name: Stirling-PDF-Security-Fat-with-login - build: - context: ../../.. - dockerfile: docker/embedded/Dockerfile.fat image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:fat build: context: ../../../ diff --git a/testing/cucumber/features/environment.py b/testing/cucumber/features/environment.py index 63cbc920e8..b8fcad1486 100644 --- a/testing/cucumber/features/environment.py +++ b/testing/cucumber/features/environment.py @@ -1,8 +1,11 @@ import os +import subprocess import requests _BASE_URL = "http://localhost:8080" +_CONTAINER_NAME = os.environ.get("TEST_CONTAINER_NAME", "") +_REPORT_DIR = os.environ.get("TEST_REPORT_DIR", "") # Tags that indicate a scenario requires JWT Bearer auth to be functional. # These scenarios are skipped when the server has JWT disabled (V2=false). @@ -44,6 +47,44 @@ def _check_jwt_available(): return False +def _get_docker_log_line_count(): + if not _CONTAINER_NAME: + return 0 + try: + result = subprocess.run( + ["docker", "logs", _CONTAINER_NAME], + capture_output=True, text=True, timeout=10, + ) + return len(result.stdout.splitlines()) + len(result.stderr.splitlines()) + except Exception: + return 0 + + +def _capture_docker_logs_window(start_line, scenario_name): + if not _CONTAINER_NAME or not _REPORT_DIR: + return + try: + result = subprocess.run( + ["docker", "logs", _CONTAINER_NAME], + capture_output=True, text=True, timeout=10, + ) + all_lines = (result.stdout + result.stderr).splitlines() + window = all_lines[start_line:] + if not window: + return + + safe_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in scenario_name) + log_path = os.path.join(_REPORT_DIR, f"scenario_{safe_name}.log") + with open(log_path, "w") as f: + f.write(f"=== Docker logs during: {scenario_name} ===\n") + f.write(f"Container: {_CONTAINER_NAME}\n") + f.write(f"Lines {start_line + 1}-{len(all_lines)} ({len(window)} lines)\n") + f.write("---\n") + f.write("\n".join(window[-200:]) + "\n") + except Exception: + pass + + def before_all(context): context.endpoint = None context.request_data = None @@ -81,8 +122,14 @@ def before_scenario(context, scenario): # Stored value used by enterprise step definitions for dynamic URL composition context.stored_value = None + context._docker_log_line_count = _get_docker_log_line_count() + def after_scenario(context, scenario): + if scenario.status == "failed": + start_line = getattr(context, "_docker_log_line_count", 0) + _capture_docker_logs_window(start_line, scenario.name) + if hasattr(context, "files"): for file in context.files.values(): try: diff --git a/testing/test.sh b/testing/test.sh index 545bafb3ff..9308cd71d3 100644 --- a/testing/test.sh +++ b/testing/test.sh @@ -31,44 +31,220 @@ find_root() { PROJECT_ROOT=$(find_root) -# Base image version - must be provided or read from environment -# This is a testing-specific version; production should pass explicit BASE_VERSION -if [ -z "$BASE_VERSION" ]; then - # For CI/automation: use a unique test identifier - if [ -n "${GITHUB_RUN_ID}" ]; then - BASE_VERSION="test-${GITHUB_RUN_ID}" +REPORT_DIR="$PROJECT_ROOT/testing/reports" +mkdir -p "$REPORT_DIR" + +declare -A test_start_times +declare -A test_durations +declare -A test_failure_logs +CURRENT_CONTAINER="" + +is_gha() { + [ -n "${GITHUB_ACTIONS:-}" ] +} + +gha_group() { + if is_gha; then echo "::group::$1"; else echo "=== $1 ==="; fi +} +gha_endgroup() { + if is_gha; then echo "::endgroup::"; fi +} + +start_test_timer() { + local test_name=$1 + test_start_times["$test_name"]=$SECONDS +} + +stop_test_timer() { + local test_name=$1 + local start=${test_start_times["$test_name"]:-$SECONDS} + test_durations["$test_name"]=$(( SECONDS - start )) +} + +capture_failure_logs() { + local test_name=$1 + local container_name=$2 + local extra_context=$3 + local log_file="$REPORT_DIR/${test_name//[^a-zA-Z0-9_-]/_}.failure.log" + + { + echo "=== Failure logs for: $test_name ===" + echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + if [ -n "$container_name" ]; then + echo "Container: $container_name" + echo "---" + docker logs "$container_name" 2>&1 | tail -200 + elif [ -n "$extra_context" ]; then + echo "---" + echo "$extra_context" + else + echo "---" + echo "No container was running. Docker-compose may have failed to start." + fi + } > "$log_file" 2>/dev/null || true + test_failure_logs["$test_name"]="$log_file" +} + +capture_build_failure() { + local build_name=$1 + local log_file="$REPORT_DIR/${build_name//[^a-zA-Z0-9_-]/_}.failure.log" + local gradle_report_dirs=( + "$PROJECT_ROOT/app/core/build/reports/tests" + "$PROJECT_ROOT/app/common/build/reports/tests" + "$PROJECT_ROOT/app/proprietary/build/reports/tests" + ) + + { + echo "=== Build failure: $build_name ===" + echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "---" + + for report_dir in "${gradle_report_dirs[@]}"; do + if [ -d "$report_dir" ]; then + local txt_index="$report_dir/test/index.html" + if [ -f "$txt_index" ]; then + echo "--- Gradle test report: $report_dir ---" + sed 's/<[^>]*>//g' "$txt_index" | head -100 + echo "" + fi + fi + done + + for xml_file in "$PROJECT_ROOT"/app/*/build/test-results/test/TEST-*.xml; do + if [ -f "$xml_file" ]; then + local failures + failures=$(grep -c 'failures="[^0]' "$xml_file" 2>/dev/null || true) + local errors + errors=$(grep -c 'errors="[^0]' "$xml_file" 2>/dev/null || true) + if [ "$failures" -gt 0 ] || [ "$errors" -gt 0 ]; then + echo "--- Failed: $(basename "$xml_file") ---" + grep -A 5 ' "$log_file" 2>/dev/null || true + + test_failure_logs["$build_name"]="$log_file" +} + + +generate_json_report() { + local report_file="$REPORT_DIR/test-report.json" + local total_duration=$SECONDS + local total_tests=$(( ${#passed_tests[@]} + ${#failed_tests[@]} )) + + { + echo "{" + echo " \"summary\": {" + echo " \"total\": $total_tests," + echo " \"passed\": ${#passed_tests[@]}," + echo " \"failed\": ${#failed_tests[@]}," + echo " \"duration_seconds\": $total_duration," + echo " \"result\": \"$([ ${#failed_tests[@]} -eq 0 ] && echo 'PASS' || echo 'FAIL')\"," + echo " \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"" + echo " }," + echo " \"tests\": [" + + local first=true + for test in "${passed_tests[@]}"; do + if [ "$first" = true ]; then first=false; else echo ","; fi + local dur=${test_durations["$test"]:-0} + printf " {\"name\": \"%s\", \"status\": \"PASS\", \"duration_seconds\": %d}" "$test" "$dur" + done + + for test in "${failed_tests[@]}"; do + if [ "$first" = true ]; then first=false; else echo ","; fi + local dur=${test_durations["$test"]:-0} + local log_file=${test_failure_logs["$test"]:-""} + local failure_snippet="" + if [ -n "$log_file" ] && [ -f "$log_file" ]; then + # Grab last 20 lines as a failure snippet, escape for JSON + failure_snippet=$(tail -20 "$log_file" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g' | tr '\n' '|' | sed 's/|/\\n/g') + fi + printf " {\"name\": \"%s\", \"status\": \"FAIL\", \"duration_seconds\": %d, \"log_file\": \"%s\", \"failure_snippet\": \"%s\"}" \ + "$test" "$dur" "$log_file" "$failure_snippet" + done + + echo "" + echo " ]" + echo "}" + } > "$report_file" + + echo "JSON test report written to: $report_file" +} + +generate_gha_summary() { + if ! is_gha; then return; fi + local summary_file="${GITHUB_STEP_SUMMARY}" + local total_duration=$SECONDS + + { + echo "## Docker Compose Test Results" + echo "" + if [ ${#failed_tests[@]} -eq 0 ]; then + echo "**Result: ALL PASSED** in ${total_duration}s" + else + echo "**Result: ${#failed_tests[@]} FAILED** out of $(( ${#passed_tests[@]} + ${#failed_tests[@]} )) tests in ${total_duration}s" + fi + echo "" + echo "| Test | Status | Duration |" + echo "|------|--------|----------|" + + for test in "${passed_tests[@]}"; do + local dur=${test_durations["$test"]:-0} + echo "| ${test} | :white_check_mark: PASS | ${dur}s |" + done + + for test in "${failed_tests[@]}"; do + local dur=${test_durations["$test"]:-0} + echo "| ${test} | :x: FAIL | ${dur}s |" + done + + # If there are failures, include snippets + if [ ${#failed_tests[@]} -ne 0 ]; then + echo "" + echo "### Failure Details" + echo "" + for test in "${failed_tests[@]}"; do + local log_file=${test_failure_logs["$test"]:-""} + echo "
" + echo "${test}" + echo "" + if [ -n "$log_file" ] && [ -f "$log_file" ]; then + echo '```' + tail -50 "$log_file" + echo '```' + else + echo "No failure logs captured. Check the full build log." + fi + echo "
" + echo "" + done + fi + } >> "$summary_file" + + echo "GitHub Actions job summary written." +} + +prepare_base_image() { + if [ "${DOCKER_BASE_CHANGED:-false}" = "true" ]; then + echo "Docker base files changed — building base image locally..." + gha_group "Build: Base image (local)" + if docker build -f "$PROJECT_ROOT/docker/base/Dockerfile" \ + -t stirling-pdf-base:local \ + "$PROJECT_ROOT/docker/base"; then + echo "✓ Built base image locally: stirling-pdf-base:local" + BASE_IMAGE_ARG="--build-arg BASE_IMAGE=stirling-pdf-base:local" + else + echo "ERROR: Failed to build base image" + gha_endgroup + return 1 + fi + gha_endgroup else - # For local testing: generate unique identifier - BASE_VERSION="test-local-$(date +%s)" - fi -fi -BASE_IMAGE="ghcr.io/stirling-tools/stirling-pdf-base:${BASE_VERSION}" - -# Function to ensure base image exists (build if missing) -ensure_base_image() { - echo "Checking for base image: $BASE_IMAGE" - - if docker image inspect "$BASE_IMAGE" >/dev/null 2>&1; then - echo "✓ Base image found locally: $BASE_IMAGE" - return 0 - fi - - echo "Base image not found. Attempting to pull from registry..." - if docker pull "$BASE_IMAGE" 2>/dev/null; then - echo "✓ Pulled base image from registry: $BASE_IMAGE" - return 0 - fi - - echo "Base image not available in registry. Building from source..." - if docker build -f "$PROJECT_ROOT/docker/base/Dockerfile" \ - -t "$BASE_IMAGE" \ - --build-arg BASE_VERSION="$BASE_VERSION" \ - "$PROJECT_ROOT/docker/base"; then - echo "✓ Built base image: $BASE_IMAGE" - return 0 - else - echo "ERROR: Failed to build base image" - return 1 + echo "Docker base unchanged — using published base image from Dockerfile defaults" + BASE_IMAGE_ARG="" fi } @@ -87,6 +263,7 @@ check_health() { echo "Using API key for health check: ${api_key:0:3}***" fi + gha_group "Health check: $container_name" echo "Waiting for $container_name to become reachable on http://localhost:8080/api/v1/info/status (timeout ${timeout}s)..." while [ $SECONDS -lt $end ]; do # Optional: check if container is running at all (nice for debugging) @@ -104,8 +281,10 @@ check_health() { # Treat any 2xx as "ready" if [ "$last_code" -ge 200 ] && [ "$last_code" -lt 300 ]; then echo "$container_name is reachable over HTTP (status $last_code)." - echo "Printing logs for $container_name:" + gha_group "Container logs (startup): $container_name" docker logs "$container_name" || true + gha_endgroup + gha_endgroup return 0 fi @@ -120,8 +299,10 @@ check_health() { docker_health=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}(no healthcheck){{end}}' "$container_name" 2>/dev/null || echo "inspect failed") echo "Docker-reported health status for $container_name: $docker_health" - echo "Printing logs for $container_name:" + gha_group "Container logs (failure): $container_name" docker logs "$container_name" || true + gha_endgroup + gha_endgroup return 1 } @@ -130,6 +311,7 @@ capture_file_list() { local container_name=$1 local output_file=$2 + gha_group "Capture file list: $container_name" echo "Capturing file list from $container_name..." # Get all files in one command, output directly from Docker to avoid path issues # Skip proc, sys, dev, and the specified LibreOffice config directory @@ -194,6 +376,7 @@ capture_file_list() { fi echo "File list captured to $output_file" + gha_endgroup } # Function to compare before and after file lists @@ -203,6 +386,7 @@ compare_file_lists() { local diff_file=$3 local container_name=$4 # Added container_name parameter + gha_group "Compare file lists" echo "Comparing file lists..." # Check if files exist and have content @@ -223,11 +407,13 @@ compare_file_lists() { cat "${diff_file}.tmp" echo "Printing docker logs due to temporary file detection:" docker logs "$container_name" # Print logs when temp files are found + gha_endgroup return 1 else echo "No temporary files found in the after snapshot." fi fi + gha_endgroup return 0 fi @@ -264,6 +450,7 @@ compare_file_lists() { else echo "No file changes detected during test." fi + gha_endgroup return 0 } @@ -313,6 +500,7 @@ test_compose() { local test_name=$2 local status=0 + gha_group "Deploy: $test_name" echo "Testing ${compose_file} configuration..." # Start up the Docker Compose service @@ -326,10 +514,16 @@ test_compose() { if [[ -z "$container_name" ]]; then echo "ERROR: No running container found for ${compose_file}" - docker-compose -f "$compose_file" ps + local compose_output + compose_output=$(docker-compose -f "$compose_file" ps 2>&1) + echo "$compose_output" + capture_failure_logs "$test_name" "" "docker-compose failed for: ${compose_file} +${compose_output}" + gha_endgroup return 1 fi + CURRENT_CONTAINER="$container_name" echo "Started container: $container_name" # Wait for the service to become healthy (HTTP-based) @@ -337,9 +531,11 @@ test_compose() { echo "${test_name} test passed." else echo "${test_name} test failed." + capture_failure_logs "$test_name" "$container_name" status=1 fi + gha_endgroup return $status } @@ -407,11 +603,19 @@ run_tests() { return 0 fi + start_test_timer "$test_name" if test_compose "$compose_file" "$test_name"; then passed_tests+=("$test_name") else failed_tests+=("$test_name") fi + stop_test_timer "$test_name" +} + +finalize_reports() { + generate_json_report + generate_gha_summary + save_failed_tests } # Main testing routine @@ -419,11 +623,12 @@ main() { SECONDS=0 cd "$PROJECT_ROOT" - # Ensure base image exists before running tests + trap finalize_reports EXIT + echo "==========================================" echo "Preparing Docker base image..." echo "==========================================" - ensure_base_image || exit 1 + prepare_base_image || exit 1 echo "" # Parse command line arguments @@ -472,9 +677,13 @@ main() { should_run_test "Webpage-Accessibility-lite" || \ should_run_test "Stirling-PDF-Ultra-Lite-Version-Check"; then + gha_group "Build: Ultra-Lite (Gradle + Docker)" export DISABLE_ADDITIONAL_FEATURES=true if ! ./gradlew clean build -PnoSpotless; then echo "Gradle build failed with security disabled, exiting script." + failed_tests+=("Build-Ultra-Lite-Gradle") + capture_build_failure "Build-Ultra-Lite-Gradle" + gha_endgroup exit 1 fi @@ -490,11 +699,16 @@ main() { else DOCKER_CACHE_ARGS_ULTRA_LITE="" fi - docker buildx build --build-arg VERSION_TAG=alpha \ + if ! docker buildx build --build-arg VERSION_TAG=alpha \ -t docker.stirlingpdf.com/stirlingtools/stirling-pdf:ultra-lite \ -f ./docker/embedded/Dockerfile.ultra-lite \ --load \ - ${DOCKER_CACHE_ARGS_ULTRA_LITE} . + ${DOCKER_CACHE_ARGS_ULTRA_LITE} .; then + failed_tests+=("Build-Ultra-Lite-Docker") + gha_endgroup + exit 1 + fi + gha_endgroup else echo "Skipping ultra-lite image build - no ultra-lite tests in rerun list" fi @@ -503,26 +717,34 @@ main() { run_tests "Stirling-PDF-Ultra-Lite" "./docker/embedded/compose/docker-compose-latest-ultra-lite.yml" if should_run_test "Webpage-Accessibility-lite"; then + start_test_timer "Webpage-Accessibility-lite" + gha_group "Test: Webpage-Accessibility-lite" echo "Testing webpage accessibility..." cd "testing" if ./test_webpages.sh -f webpage_urls.txt -b http://localhost:8080; then passed_tests+=("Webpage-Accessibility-lite") else failed_tests+=("Webpage-Accessibility-lite") + capture_failure_logs "Webpage-Accessibility-lite" "$CURRENT_CONTAINER" echo "Webpage accessibility lite tests failed" fi cd "$PROJECT_ROOT" + gha_endgroup + stop_test_timer "Webpage-Accessibility-lite" fi if should_run_test "Stirling-PDF-Ultra-Lite-Version-Check"; then + start_test_timer "Stirling-PDF-Ultra-Lite-Version-Check" echo "Testing version verification..." if verify_app_version "Stirling-PDF-Ultra-Lite" "http://localhost:8080"; then passed_tests+=("Stirling-PDF-Ultra-Lite-Version-Check") echo "Version verification passed for Stirling-PDF-Ultra-Lite" else failed_tests+=("Stirling-PDF-Ultra-Lite-Version-Check") + capture_failure_logs "Stirling-PDF-Ultra-Lite-Version-Check" "$CURRENT_CONTAINER" echo "Version verification failed for Stirling-PDF-Ultra-Lite" fi + stop_test_timer "Stirling-PDF-Ultra-Lite-Version-Check" fi docker-compose -f "./docker/embedded/compose/docker-compose-latest-ultra-lite.yml" down -v @@ -540,9 +762,13 @@ main() { should_run_test "Disabled-Endpoints" || \ should_run_test "Stirling-PDF-Fat-Disable-Endpoints-Version-Check"; then + gha_group "Build: Fat + Security (Gradle + Docker)" export DISABLE_ADDITIONAL_FEATURES=false if ! ./gradlew clean build -PnoSpotless; then echo "Gradle build failed with security enabled, exiting script." + failed_tests+=("Build-Fat-Gradle") + capture_build_failure "Build-Fat-Gradle" + gha_endgroup exit 1 fi @@ -557,11 +783,17 @@ main() { else DOCKER_CACHE_ARGS_FAT="" fi - docker buildx build --build-arg VERSION_TAG=alpha \ + if ! docker buildx build --build-arg VERSION_TAG=alpha \ + ${BASE_IMAGE_ARG} \ -t docker.stirlingpdf.com/stirlingtools/stirling-pdf:fat \ -f ./docker/embedded/Dockerfile.fat \ --load \ - ${DOCKER_CACHE_ARGS_FAT} . + ${DOCKER_CACHE_ARGS_FAT} .; then + failed_tests+=("Build-Fat-Docker") + gha_endgroup + exit 1 + fi + gha_endgroup else echo "Skipping fat image build - no fat tests in rerun list" fi @@ -570,26 +802,34 @@ main() { run_tests "Stirling-PDF-Security-Fat" "./docker/embedded/compose/docker-compose-latest-fat-security.yml" if should_run_test "Webpage-Accessibility-full"; then + start_test_timer "Webpage-Accessibility-full" + gha_group "Test: Webpage-Accessibility-full" echo "Testing webpage accessibility..." cd "testing" if ./test_webpages.sh -f webpage_urls_full.txt -b http://localhost:8080; then passed_tests+=("Webpage-Accessibility-full") else failed_tests+=("Webpage-Accessibility-full") + capture_failure_logs "Webpage-Accessibility-full" "$CURRENT_CONTAINER" echo "Webpage accessibility full tests failed" fi cd "$PROJECT_ROOT" + gha_endgroup + stop_test_timer "Webpage-Accessibility-full" fi if should_run_test "Stirling-PDF-Security-Fat-Version-Check"; then + start_test_timer "Stirling-PDF-Security-Fat-Version-Check" echo "Testing version verification..." if verify_app_version "Stirling-PDF-Security-Fat" "http://localhost:8080"; then passed_tests+=("Stirling-PDF-Security-Fat-Version-Check") echo "Version verification passed for Stirling-PDF-Security-Fat" else failed_tests+=("Stirling-PDF-Security-Fat-Version-Check") + capture_failure_logs "Stirling-PDF-Security-Fat-Version-Check" "$CURRENT_CONTAINER" echo "Version verification failed for Stirling-PDF-Security-Fat" fi + stop_test_timer "Stirling-PDF-Security-Fat-Version-Check" fi docker-compose -f "./docker/embedded/compose/docker-compose-latest-fat-security.yml" down -v @@ -617,10 +857,24 @@ main() { CUCUMBER_JUNIT_DIR="$PROJECT_ROOT/testing/cucumber/junit" mkdir -p "$CUCUMBER_JUNIT_DIR" cd "testing/cucumber" + start_test_timer "Stirling-PDF-Regression" + + # Snapshot docker log line count before behave so we can extract only behave-window logs + DOCKER_LOG_BEFORE=$(docker logs "$CONTAINER_NAME" 2>&1 | wc -l) + + export TEST_CONTAINER_NAME="$CONTAINER_NAME" + export TEST_REPORT_DIR="$REPORT_DIR" + + gha_group "Test: Behave regression tests" if python -m behave \ -f behave_html_formatter:HTMLFormatter -o "$CUCUMBER_REPORT" \ -f pretty \ --junit --junit-directory "$CUCUMBER_JUNIT_DIR"; then + gha_endgroup + + # Save docker logs produced during the behave run + docker logs "$CONTAINER_NAME" 2>&1 | tail -n +"$((DOCKER_LOG_BEFORE + 1))" > "$REPORT_DIR/cucumber-docker-context.log" 2>/dev/null || true + echo "Waiting 5 seconds for any file operations to complete..." sleep 5 @@ -636,10 +890,17 @@ main() { fi passed_tests+=("Stirling-PDF-Regression $CONTAINER_NAME") else + gha_endgroup failed_tests+=("Stirling-PDF-Regression $CONTAINER_NAME") - echo "Printing docker logs of failed regression" - docker logs "$CONTAINER_NAME" - echo "Printed docker logs of failed regression" + + # Save docker logs from the behave window to a dedicated file + local cucumber_log="$REPORT_DIR/cucumber-docker-context.log" + docker logs "$CONTAINER_NAME" 2>&1 | tail -n +"$((DOCKER_LOG_BEFORE + 1))" > "$cucumber_log" 2>/dev/null || true + test_failure_logs["Stirling-PDF-Regression"]="$cucumber_log" + + gha_group "Docker logs during behave run: $CONTAINER_NAME" + tail -100 "$cucumber_log" + gha_endgroup echo "Waiting 10 seconds before capturing file list..." sleep 10 @@ -648,6 +909,7 @@ main() { capture_file_list "$CONTAINER_NAME" "$AFTER_FILE" compare_file_lists "$BEFORE_FILE" "$AFTER_FILE" "$DIFF_FILE" "$CONTAINER_NAME" fi + stop_test_timer "Stirling-PDF-Regression" fi docker-compose -f "./docker/embedded/compose/test_cicd.yml" down -v @@ -657,24 +919,32 @@ main() { run_tests "Stirling-PDF-Fat-Disable-Endpoints" "./docker/embedded/compose/docker-compose-latest-fat-endpoints-disabled.yml" if should_run_test "Disabled-Endpoints"; then + start_test_timer "Disabled-Endpoints" + gha_group "Test: Disabled-Endpoints" echo "Testing disabled endpoints..." if ./testing/test_disabledEndpoints.sh -f ./testing/endpoints.txt -b http://localhost:8080; then passed_tests+=("Disabled-Endpoints") else failed_tests+=("Disabled-Endpoints") + capture_failure_logs "Disabled-Endpoints" "$CURRENT_CONTAINER" echo "Disabled Endpoints tests failed" fi + gha_endgroup + stop_test_timer "Disabled-Endpoints" fi if should_run_test "Stirling-PDF-Fat-Disable-Endpoints-Version-Check"; then + start_test_timer "Stirling-PDF-Fat-Disable-Endpoints-Version-Check" echo "Testing version verification..." if verify_app_version "Stirling-PDF-Fat-Disable-Endpoints" "http://localhost:8080"; then passed_tests+=("Stirling-PDF-Fat-Disable-Endpoints-Version-Check") echo "Version verification passed for Stirling-PDF-Fat-Disable-Endpoints" else failed_tests+=("Stirling-PDF-Fat-Disable-Endpoints-Version-Check") + capture_failure_logs "Stirling-PDF-Fat-Disable-Endpoints-Version-Check" "$CURRENT_CONTAINER" echo "Version verification failed for Stirling-PDF-Fat-Disable-Endpoints" fi + stop_test_timer "Stirling-PDF-Fat-Disable-Endpoints-Version-Check" fi docker-compose -f "./docker/embedded/compose/docker-compose-latest-fat-endpoints-disabled.yml" down -v @@ -682,24 +952,33 @@ main() { # ================================================================== # Final Report # ================================================================== - echo "All tests completed in $SECONDS seconds." + echo "" + echo "==========================================" + echo "TEST RESULTS SUMMARY" + echo "==========================================" + echo "Total duration: ${SECONDS}s" + echo "Passed: ${#passed_tests[@]} Failed: ${#failed_tests[@]}" + echo "" if [ ${#passed_tests[@]} -ne 0 ]; then echo "Passed tests:" for test in "${passed_tests[@]}"; do - echo -e "\e[32m$test\e[0m" + local dur=${test_durations["$test"]:-"?"} + echo -e " \e[32m✅ $test\e[0m (${dur}s)" done fi if [ ${#failed_tests[@]} -ne 0 ]; then + echo "" echo "Failed tests:" for test in "${failed_tests[@]}"; do - echo -e "\e[31m$test\e[0m" + local dur=${test_durations["$test"]:-"?"} + local log=${test_failure_logs["$test"]:-"no log captured"} + echo -e " \e[31m❌ $test\e[0m (${dur}s) -> $log" done fi - # Save failed tests for potential rerun - save_failed_tests + echo "" if [ ${#failed_tests[@]} -ne 0 ]; then echo "Some tests failed."