diff --git a/Dockerfile b/Dockerfile index fd7855e..5d30778 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,35 +7,47 @@ FROM golang:1.21 AS builder WORKDIR /build -# Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ +# Install dependencies (optimized for caching) +RUN --mount=type=cache,target=/var/cache/apt \ + --mount=type=cache,target=/var/lib/apt/lists \ + apt-get update && apt-get install -y --no-install-recommends \ git \ bc \ && rm -rf /var/lib/apt/lists/* # Copy go mod files -COPY go.mod ./ +COPY go.mod go.sum ./ -# Download dependencies -RUN go mod download || true +# Download dependencies (cached layer) +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download -# Copy source code +# Copy source code (this layer changes frequently) COPY . . -# Run tests with coverage (fail if < 80%) -RUN go test -race -coverprofile=coverage.txt -covermode=atomic ./... && \ - echo "=== Coverage Report ===" && \ - go tool cover -func=coverage.txt | grep total && \ - TOTAL=$(go tool cover -func=coverage.txt | grep total | awk '{gsub(/%/, "", $3); print $3}') && \ - echo "Total coverage: ${TOTAL}%" && \ - if (( $(echo "$TOTAL < 80" | bc -l) )); then \ - echo "ERROR: Coverage ${TOTAL}% is below 80% threshold"; \ - exit 1; \ - fi && \ - echo "Coverage check passed!" +# ARG to skip tests (for faster builds) +ARG SKIP_TESTS=false + +# Run tests with coverage (fail if < 80%) - can be skipped with SKIP_TESTS=true +RUN --mount=type=cache,target=/go/pkg/mod \ + if [ "$SKIP_TESTS" = "false" ]; then \ + go test -race -coverprofile=coverage.txt -covermode=atomic ./... && \ + echo "=== Coverage Report ===" && \ + go tool cover -func=coverage.txt | grep total && \ + TOTAL=$(go tool cover -func=coverage.txt | grep total | awk '{gsub(/%/, "", $3); print $3}') && \ + echo "Total coverage: ${TOTAL}%" && \ + if (( $(echo "$TOTAL < 80" | bc -l) )); then \ + echo "ERROR: Coverage ${TOTAL}% is below 80% threshold"; \ + exit 1; \ + fi && \ + echo "Coverage check passed!"; \ + else \ + echo "Skipping tests (SKIP_TESTS=true)"; \ + fi # Build binary -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ +RUN --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -ldflags="-w -s" \ -o /usr/bin/logcorrelator \ ./cmd/logcorrelator diff --git a/Dockerfile.package b/Dockerfile.package index ee59c84..92e0511 100644 --- a/Dockerfile.package +++ b/Dockerfile.package @@ -1,30 +1,147 @@ # syntax=docker/dockerfile:1 # ============================================================================= # logcorrelator - Dockerfile de build et packaging RPM multi-distros +# Optimisé avec stages communs et builds parallèles # ============================================================================= # ============================================================================= -# Stage 1: Builder - Compilation du binaire Go +# Stage 0: Common RPM tools - Shared across all distributions +# ============================================================================= +FROM ruby:3.2-bookworm AS rpm-common-tools + +WORKDIR /package + +# Install RPM build tools and fpm (COMMON - cached across all distros) +RUN --mount=type=cache,target=/var/cache/dnf \ + --mount=type=cache,target=/var/cache/ruby \ + dnf install -y epel-release && \ + dnf install -y ruby rubygems ruby-devel rpm-build gcc make -y && \ + gem install fpm -v 1.16.0 --no-document && \ + dnf clean all + +# Common script to build RPM (parameterized) +COPY <" \ + --rpm-dist \${DIST_NAME} \ + --depends "systemd" \ + --after-install /tmp/scripts/post \ + --before-remove /tmp/scripts/preun \ + --after-remove /tmp/scripts/postun \ + -p /packages/rpm/\${DIST_NAME}/logcorrelator-\${VERSION}-1.\${DIST_NAME}.x86_64.rpm \ + usr/bin/logcorrelator \ + etc/logcorrelator/logcorrelator.yml \ + etc/logcorrelator/logcorrelator.yml.example \ + var/log/logcorrelator \ + var/run/logcorrelator \ + var/lib/logcorrelator \ + etc/systemd/system/logcorrelator.service \ + etc/logrotate.d/logcorrelator + +echo "RPM built for \${DIST_NAME}" +EOF + +RUN chmod +x /build-rpm.sh + +# ============================================================================= +# Stage 1: Builder - Compilation du binaire Go (shared by all RPM builds) # ============================================================================= FROM golang:1.21 AS builder WORKDIR /build -# Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ +# Install dependencies (minimal, just for Go build) +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* # Copy go mod files COPY go.mod go.sum ./ -RUN go mod download + +# Download dependencies (cached) +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download # Copy source code COPY . . # Build binary for Linux -ARG VERSION=$(grep -m1 "^Version:" packaging/rpm/logcorrelator.spec | awk '{print $2}') -RUN mkdir -p dist && \ +ARG VERSION=1.0.0 +RUN --mount=type=cache,target=/go/pkg/mod \ + mkdir -p dist && \ CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ go build -ldflags="-w -s -X main.Version=${VERSION}" \ -o dist/logcorrelator \ @@ -32,209 +149,48 @@ RUN mkdir -p dist && \ # ============================================================================= # Stage 2: RPM Package builder for Enterprise Linux 8 (el8) +# Uses common RPM tools from rpm-common-tools stage # ============================================================================= -FROM rockylinux:8 AS rpm-el8-builder +FROM rpm-common-tools AS rpm-el8-builder WORKDIR /package -# Install RPM build tools and fpm -RUN dnf install -y epel-release && \ - dnf install -y ruby rubygems ruby-devel rpm-build gcc make && \ - gem install fpm -v 1.16.0 --no-document && \ - dnf clean all +# Copy builder stage for binary +COPY --from=builder /build /build -# Copy binary from builder -COPY --from=builder /build/dist/logcorrelator /tmp/pkgroot/usr/bin/logcorrelator -# Config files: .yml is marked %config(noreplace) in RPM spec (preserved on upgrade) -# .yml.example is always updated to reflect latest configuration options -COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml -COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml.example -COPY --from=builder /build/logcorrelator.service /tmp/pkgroot/etc/systemd/system/logcorrelator.service -COPY packaging/rpm/post /tmp/scripts/post -COPY packaging/rpm/preun /tmp/scripts/preun -COPY packaging/rpm/postun /tmp/scripts/postun -COPY packaging/rpm/logrotate /tmp/pkgroot/etc/logrotate.d/logcorrelator - -# Create directories and set permissions -# /var/run/logcorrelator: 755 - will be owned by logcorrelator:logcorrelator by post install script -# /var/log/logcorrelator: 755 - will be owned by logcorrelator:logcorrelator by post install script -# /var/lib/logcorrelator: created for service home directory -RUN mkdir -p /tmp/pkgroot/var/log/logcorrelator && \ - mkdir -p /tmp/pkgroot/var/run/logcorrelator && \ - mkdir -p /tmp/pkgroot/var/lib/logcorrelator && \ - chmod 755 /tmp/pkgroot/usr/bin/logcorrelator && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml.example && \ - chmod 644 /tmp/pkgroot/etc/systemd/system/logcorrelator.service && \ - chmod 755 /tmp/scripts/* && \ - chmod 755 /tmp/pkgroot/var/log/logcorrelator && \ - chmod 755 /tmp/pkgroot/var/run/logcorrelator && \ - chmod 755 /tmp/pkgroot/var/lib/logcorrelator - -# Build RPM for Enterprise Linux 8 (el8) -# Note: fpm does not support %config(noreplace) directly; this is handled in the spec file -# The post install script ensures existing config is preserved -ARG VERSION=$(grep -m1 "^Version:" packaging/rpm/logcorrelator.spec | awk '{print $2}') -RUN mkdir -p /packages/rpm/el8 && \ - fpm -s dir -t rpm \ - -n logcorrelator \ - -v "${VERSION}" \ - -C /tmp/pkgroot \ - --architecture "x86_64" \ - --rpm-os linux \ - --description "Log correlation service for HTTP and network events" \ - --url "https://github.com/logcorrelator/logcorrelator" \ - --license "MIT" \ - --vendor "logcorrelator " \ - --rpm-dist el8 \ - --depends "systemd" \ - --after-install /tmp/scripts/post \ - --before-remove /tmp/scripts/preun \ - --after-remove /tmp/scripts/postun \ - -p /packages/rpm/el8/logcorrelator-${VERSION}-1.el8.x86_64.rpm \ - usr/bin/logcorrelator \ - etc/logcorrelator/logcorrelator.yml \ - etc/logcorrelator/logcorrelator.yml.example \ - var/log/logcorrelator \ - var/run/logcorrelator \ - etc/systemd/system/logcorrelator.service \ - etc/logrotate.d/logcorrelator +# Build RPM for el8 +ARG VERSION=1.0.0 +RUN /build-rpm.sh el8 rockylinux:8 ${VERSION} # ============================================================================= # Stage 3: RPM Package builder for Enterprise Linux 9 (el9) +# Uses common RPM tools from rpm-common-tools stage # ============================================================================= -FROM rockylinux:9 AS rpm-el9-builder +FROM rpm-common-tools AS rpm-el9-builder WORKDIR /package -# Install RPM build tools and fpm -RUN dnf install -y epel-release && \ - dnf install -y ruby rubygems ruby-devel rpm-build gcc make && \ - gem install fpm -v 1.16.0 --no-document && \ - dnf clean all +# Copy builder stage for binary +COPY --from=builder /build /build -# Copy binary from builder -COPY --from=builder /build/dist/logcorrelator /tmp/pkgroot/usr/bin/logcorrelator -# Config files: .yml is marked %config(noreplace) in RPM spec (preserved on upgrade) -# .yml.example is always updated to reflect latest configuration options -COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml -COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml.example -COPY --from=builder /build/logcorrelator.service /tmp/pkgroot/etc/systemd/system/logcorrelator.service -COPY packaging/rpm/post /tmp/scripts/post -COPY packaging/rpm/preun /tmp/scripts/preun -COPY packaging/rpm/postun /tmp/scripts/postun -COPY packaging/rpm/logrotate /tmp/pkgroot/etc/logrotate.d/logcorrelator - -# Create directories and set permissions -# /var/run/logcorrelator: 755 - will be owned by logcorrelator:logcorrelator by post install script -# /var/log/logcorrelator: 755 - will be owned by logcorrelator:logcorrelator by post install script -# /var/lib/logcorrelator: created for service home directory -RUN mkdir -p /tmp/pkgroot/var/log/logcorrelator && \ - mkdir -p /tmp/pkgroot/var/run/logcorrelator && \ - mkdir -p /tmp/pkgroot/var/lib/logcorrelator && \ - chmod 755 /tmp/pkgroot/usr/bin/logcorrelator && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml.example && \ - chmod 644 /tmp/pkgroot/etc/systemd/system/logcorrelator.service && \ - chmod 755 /tmp/scripts/* && \ - chmod 755 /tmp/pkgroot/var/log/logcorrelator && \ - chmod 755 /tmp/pkgroot/var/run/logcorrelator && \ - chmod 755 /tmp/pkgroot/var/lib/logcorrelator - -# Build RPM for Enterprise Linux 9 (el9) -ARG VERSION=$(grep -m1 "^Version:" packaging/rpm/logcorrelator.spec | awk '{print $2}') -RUN mkdir -p /packages/rpm/el9 && \ - fpm -s dir -t rpm \ - -n logcorrelator \ - -v "${VERSION}" \ - -C /tmp/pkgroot \ - --architecture "x86_64" \ - --rpm-os linux \ - --description "Log correlation service for HTTP and network events" \ - --url "https://github.com/logcorrelator/logcorrelator" \ - --license "MIT" \ - --vendor "logcorrelator " \ - --rpm-dist el9 \ - --depends "systemd" \ - --after-install /tmp/scripts/post \ - --before-remove /tmp/scripts/preun \ - --after-remove /tmp/scripts/postun \ - -p /packages/rpm/el9/logcorrelator-${VERSION}-1.el9.x86_64.rpm \ - usr/bin/logcorrelator \ - etc/logcorrelator/logcorrelator.yml \ - etc/logcorrelator/logcorrelator.yml.example \ - var/log/logcorrelator \ - var/run/logcorrelator \ - etc/systemd/system/logcorrelator.service \ - etc/logrotate.d/logcorrelator +# Build RPM for el9 +ARG VERSION=1.0.0 +RUN /build-rpm.sh el9 rockylinux:9 ${VERSION} # ============================================================================= # Stage 4: RPM Package builder for Enterprise Linux 10 (el10) +# Uses common RPM tools from rpm-common-tools stage # ============================================================================= -FROM almalinux:10 AS rpm-el10-builder +FROM rpm-common-tools AS rpm-el10-builder WORKDIR /package -# Install RPM build tools and fpm -RUN dnf install -y epel-release && \ - dnf install -y ruby rubygems ruby-devel rpm-build gcc make && \ - gem install fpm -v 1.16.0 --no-document && \ - dnf clean all +# Copy builder stage for binary +COPY --from=builder /build /build -# Copy binary from builder -COPY --from=builder /build/dist/logcorrelator /tmp/pkgroot/usr/bin/logcorrelator -# Config files: .yml is marked %config(noreplace) in RPM spec (preserved on upgrade) -# .yml.example is always updated to reflect latest configuration options -COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml -COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml.example -COPY --from=builder /build/logcorrelator.service /tmp/pkgroot/etc/systemd/system/logcorrelator.service -COPY packaging/rpm/post /tmp/scripts/post -COPY packaging/rpm/preun /tmp/scripts/preun -COPY packaging/rpm/postun /tmp/scripts/postun -COPY packaging/rpm/logrotate /tmp/pkgroot/etc/logrotate.d/logcorrelator - -# Create directories and set permissions -# /var/run/logcorrelator: 755 - will be owned by logcorrelator:logcorrelator by post install script -# /var/log/logcorrelator: 755 - will be owned by logcorrelator:logcorrelator by post install script -# /var/lib/logcorrelator: created for service home directory -RUN mkdir -p /tmp/pkgroot/var/log/logcorrelator && \ - mkdir -p /tmp/pkgroot/var/run/logcorrelator && \ - mkdir -p /tmp/pkgroot/var/lib/logcorrelator && \ - chmod 755 /tmp/pkgroot/usr/bin/logcorrelator && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml.example && \ - chmod 644 /tmp/pkgroot/etc/systemd/system/logcorrelator.service && \ - chmod 755 /tmp/scripts/* && \ - chmod 755 /tmp/pkgroot/var/log/logcorrelator && \ - chmod 755 /tmp/pkgroot/var/run/logcorrelator && \ - chmod 755 /tmp/pkgroot/var/lib/logcorrelator - -# Build RPM for Enterprise Linux 10 (el10) -ARG VERSION=$(grep -m1 "^Version:" packaging/rpm/logcorrelator.spec | awk '{print $2}') -RUN mkdir -p /packages/rpm/el10 && \ - fpm -s dir -t rpm \ - -n logcorrelator \ - -v "${VERSION}" \ - -C /tmp/pkgroot \ - --architecture "x86_64" \ - --rpm-os linux \ - --description "Log correlation service for HTTP and network events" \ - --url "https://github.com/logcorrelator/logcorrelator" \ - --license "MIT" \ - --vendor "logcorrelator " \ - --rpm-dist el10 \ - --depends "systemd" \ - --after-install /tmp/scripts/post \ - --before-remove /tmp/scripts/preun \ - --after-remove /tmp/scripts/postun \ - -p /packages/rpm/el10/logcorrelator-${VERSION}-1.el10.x86_64.rpm \ - usr/bin/logcorrelator \ - etc/logcorrelator/logcorrelator.yml \ - etc/logcorrelator/logcorrelator.yml.example \ - var/log/logcorrelator \ - var/run/logcorrelator \ - etc/systemd/system/logcorrelator.service \ - etc/logrotate.d/logcorrelator +# Build RPM for el10 +ARG VERSION=1.0.0 +RUN /build-rpm.sh el10 almalinux:10 ${VERSION} # ============================================================================= # Stage 5: Output - Image finale avec les packages RPM diff --git a/Makefile b/Makefile index d26b273..b4e2126 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,18 @@ # Docker parameters DOCKER=docker -DOCKER_BUILD=$(DOCKER) build --no-cache +# Use buildx for better cache management and parallel builds +DOCKER_BUILD=$(DOCKER) build +DOCKER_BUILDX=$(DOCKER) buildx DOCKER_RUN=$(DOCKER) run # Image names DEV_IMAGE=logcorrelator-dev:latest RUNTIME_IMAGE=logcorrelator:latest PACKAGER_IMAGE=logcorrelator-packager:latest +PACKAGER_IMAGE_EL8=logcorrelator-packager-el8:latest +PACKAGER_IMAGE_EL9=logcorrelator-packager-el9:latest +PACKAGER_IMAGE_EL10=logcorrelator-packager-el10:latest # Binary name BINARY_NAME=logcorrelator @@ -17,16 +22,23 @@ DIST_DIR=dist # Package version PKG_VERSION ?= 1.1.7 +# Enable BuildKit for better performance +export DOCKER_BUILDKIT=1 + ## build: Build the logcorrelator binary locally build: mkdir -p $(DIST_DIR) go build -ldflags="-w -s" -o $(DIST_DIR)/$(BINARY_NAME) ./cmd/$(BINARY_NAME) -## docker-build-dev: Build the development Docker image +## docker-build-dev: Build the development Docker image (with tests and coverage) docker-build-dev: $(DOCKER_BUILD) --target builder -t $(DEV_IMAGE) -f Dockerfile . -## docker-build-runtime: Build the runtime Docker image +## docker-build-dev-no-test: Build the development Docker image WITHOUT tests (faster) +docker-build-dev-no-test: + $(DOCKER_BUILD) --target builder --no-cache --build-arg SKIP_TESTS=true -t $(DEV_IMAGE) -f Dockerfile . + +## docker-build-runtime: Build the runtime Docker image (fast, no tests) docker-build-runtime: $(DOCKER_BUILD) --target runtime -t $(RUNTIME_IMAGE) -f Dockerfile . @@ -51,11 +63,15 @@ fmt: package: package-rpm ## package-rpm: Build RPM packages for Rocky Linux 8/9, AlmaLinux 10 (requires Docker) +## Uses buildx for parallel builds (el8, el9, el10 built simultaneously) package-rpm: mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10 - $(DOCKER_BUILD) --target output -t $(PACKAGER_IMAGE) \ + @echo "Starting parallel RPM builds for el8, el9, el10..." + # Build all three distributions in parallel using buildx + $(DOCKER_BUILDX) build --target output -t $(PACKAGER_IMAGE) \ --build-arg VERSION=$(PKG_VERSION) \ - -f Dockerfile.package . + -f Dockerfile.package . \ + --load @echo "Extracting RPM packages from Docker image..." $(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm $(PACKAGER_IMAGE) sh -c \ "cp -r /packages/rpm/el8 /output/rpm/ && \ @@ -69,6 +85,31 @@ package-rpm: @echo " Enterprise Linux 10 (el10):" ls -la $(DIST_DIR)/rpm/el10/ 2>/dev/null || echo " (no packages)" +## package-rpm-sequential: Build RPM packages sequentially (fallback if parallel fails) +package-rpm-sequential: + mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10 + @echo "Building RPM for el8..." + $(DOCKER_BUILD) --target rpm-el8-builder -t $(PACKAGER_IMAGE_EL8) \ + --build-arg VERSION=$(PKG_VERSION) \ + -f Dockerfile.package . + @echo "Building RPM for el9..." + $(DOCKER_BUILD) --target rpm-el9-builder -t $(PACKAGER_IMAGE_EL9) \ + --build-arg VERSION=$(PKG_VERSION) \ + -f Dockerfile.package . + @echo "Building RPM for el10..." + $(DOCKER_BUILD) --target rpm-el10-builder -t $(PACKAGER_IMAGE_EL10) \ + --build-arg VERSION=$(PKG_VERSION) \ + -f Dockerfile.package . + @echo "Extracting RPM packages..." + $(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm \ + -v $(PACKAGER_IMAGE_EL8):/el8:ro \ + -v $(PACKAGER_IMAGE_EL9):/el9:ro \ + -v $(PACKAGER_IMAGE_EL10):/el10:ro \ + alpine:latest sh -c \ + "cp -r /el8/packages/rpm/el8 /output/rpm/ && \ + cp -r /el9/packages/rpm/el9 /output/rpm/ && \ + cp -r /el10/packages/rpm/el10 /output/rpm/" + ## test-package-rpm: Test RPM package installation in Docker test-package-rpm: package-rpm ./packaging/test/test-rpm.sh