.PHONY: build venv clean redpanda-git treesitter generate-docs check # Default tag (can be overridden via `make TAG=v25.1.1`) TAG ?= dev # Derive a β€œmajor.minor” or rc identifier from TAG for folder naming VERSION := $(shell \ if echo "$(TAG)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \ echo "$(TAG)" \ | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \ | sed 's/-rc/rc/'; \ else \ echo "$(TAG)"; \ fi) # Paths REPO_ROOT := $(shell git rev-parse --show-toplevel) MODULE_ROOT := $(shell cd "$(dir $(realpath $(lastword $(MAKEFILE_LIST))))"/../.. && pwd) TOOL_ROOT := $(MODULE_ROOT)/tools/property-extractor TMP_ROOT := $(TOOL_ROOT)/tmp REDPANDA_SRC := $(TMP_ROOT)/redpanda TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp VENV := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv PYTHON := $(VENV)/bin/python TREE_SITTER := npx tree-sitter # Output directory configuration (can be overridden by environment variables) OUTPUT_AUTOGENERATED_DIR ?= $(REPO_ROOT)/modules/reference OUTPUT_ASCIIDOC_DIR ?= $(OUTPUT_AUTOGENERATED_DIR)/pages # Always save versioned JSON files to attachments directory OUTPUT_JSON_DIR ?= $(OUTPUT_AUTOGENERATED_DIR)/attachments # Diff JSON files are saved to overrides directory (if OVERRIDES is specified) OUTPUT_DIFF_DIR ?= $(if $(OVERRIDES),$(or $(dir $(OVERRIDES)),$(OUTPUT_AUTOGENERATED_DIR)/attachments),$(OUTPUT_AUTOGENERATED_DIR)/attachments) OUTPUT_PARTIALS_DIR ?= $(OUTPUT_AUTOGENERATED_DIR)/partials # --- Main build: venv, fetch code, build parser, extract & docgen --- build: venv redpanda-git treesitter @echo "πŸ”§ Building with Redpanda tag: $(TAG)" @mkdir -p $(TOOL_ROOT)/gen @if [ -n "$(OVERRIDES)" ] && [ ! -f "$(OVERRIDES)" ]; then \ echo "❌ Error: Overrides file '$(OVERRIDES)' does not exist."; \ echo " Please check the path or omit the OVERRIDES variable to continue without overrides."; \ exit 1; \ fi @cd $(TOOL_ROOT) && \ $(PYTHON) -W ignore property_extractor.py \ --recursive \ --path $(REDPANDA_SRC) \ --output gen/properties-output.json \ --enhanced-output gen/$(TAG)-properties.json \ $(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),) \ $(if $(CLOUD_SUPPORT),--cloud-support,) @echo "βœ… JSON generated at $(TOOL_ROOT)/gen/properties-output.json" @echo "βœ… Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json" @$(MAKE) generate-docs # --- Ensure Python venv & dependencies --- venv: $(TOOL_ROOT)/requirements.txt @if [ ! -d "$(VENV)" ]; then \ echo "🐍 Creating virtual environment in $(VENV)..."; \ python3 -m venv $(VENV); \ else \ echo "🐍 Virtual environment already exists at $(VENV)"; \ fi; \ echo "πŸ”„ Upgrading pip and installing requirements..."; \ $(VENV)/bin/pip install --upgrade pip --quiet; \ $(VENV)/bin/pip install --no-cache-dir -r $< --quiet; # --- Clean out all generated state --- clean: @echo "🧹 Cleaning up…" @rm -rf $(VENV) $(TMP_ROOT) $(REPO_ROOT)/autogenerated # --- Clone or update Redpanda, checkout TAG or default branch --- redpanda-git: @mkdir -p "$(TMP_ROOT)" @echo "πŸ”„ Cloning/updating Redpanda into $(REDPANDA_SRC)…" @if [ -d "$(REDPANDA_SRC)" ]; then \ git -C "$(REDPANDA_SRC)" fetch --all --tags -q; \ else \ GH_TOKEN=$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN:-$${GH_TOKEN}}}; \ if [ -n "$$GH_TOKEN" ]; then \ echo "πŸ”‘ Using authenticated clone (token provided)"; \ git clone -q https://$$GH_TOKEN@github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \ else \ git clone -q https://github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \ fi; \ fi; \ if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "origin/$(TAG)" >/dev/null 2>&1; then \ echo "πŸ”– Checking out remote branch 'origin/$(TAG)'"; \ git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)" 2>/dev/null || git -C "$(REDPANDA_SRC)" checkout -q -b "$(TAG)" "origin/$(TAG)"; \ git -C "$(REDPANDA_SRC)" reset --hard "origin/$(TAG)" -q; \ elif git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null 2>&1; then \ echo "πŸ”– Checking out '$(TAG)'"; \ git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)"; \ else \ DEFAULT_BRANCH=$$(git -C "$(REDPANDA_SRC)" symbolic-ref --short refs/remotes/origin/HEAD); \ echo "⚠️ Tag '$(TAG)' not found; falling back to '$${DEFAULT_BRANCH}'"; \ git -C "$(REDPANDA_SRC)" checkout -q "$${DEFAULT_BRANCH}"; \ fi # --- Clone Tree-sitter grammar & generate parser --- treesitter: @echo "🌲 Ensuring tree-sitter-cpp grammar…" @if [ ! -d "$(TREESITTER_DIR)/.git" ]; then \ rm -rf "$(TREESITTER_DIR)"; \ git clone https://github.com/tree-sitter/tree-sitter-cpp.git "$(TREESITTER_DIR)"; \ fi @echo "πŸ”„ Ensuring tree-sitter-cpp is at compatible version v0.20.5…" @cd "$(TREESITTER_DIR)" && \ git fetch --tags -q && \ git checkout -q v0.20.5 @echo "πŸ”§ Generating parser in $(TREESITTER_DIR)…" @cd "$(TREESITTER_DIR)" && export CFLAGS="-Wno-unused-but-set-variable" && npm install --silent && export CFLAGS="-Wno-unused-but-set-variable" && $(TREE_SITTER) generate # --- Install Node.js dependencies for Handlebars --- node-deps: @echo "πŸ“¦ Installing Node.js dependencies…" @cd $(TOOL_ROOT) && npm install --silent # --- Turn the JSON into AsciiDoc pages under the specified output directories --- generate-docs: node-deps @echo "πŸ“ Generating AsciiDoc pages in $(OUTPUT_ASCIIDOC_DIR)…" @mkdir -p "$(OUTPUT_ASCIIDOC_DIR)" @mkdir -p "$(OUTPUT_JSON_DIR)" @# Use the enhanced properties file (with overrides) for documentation generation if it exists @if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \ cd $(TOOL_ROOT) && \ node generate-handlebars-docs.js "gen/$(TAG)-properties.json" "$(OUTPUT_ASCIIDOC_DIR)"; \ else \ cd $(TOOL_ROOT) && \ node generate-handlebars-docs.js "gen/properties-output.json" "$(OUTPUT_ASCIIDOC_DIR)"; \ fi @echo "πŸ“„ Copying properties JSON files to $(OUTPUT_JSON_DIR)…" @if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \ echo " Source: $(TOOL_ROOT)/gen/$(TAG)-properties.json"; \ echo " Target: $(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json"; \ cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json" && \ echo " βœ… Successfully copied redpanda-properties-$(TAG).json" || \ { echo " ❌ Failed to copy redpanda-properties-$(TAG).json"; exit 1; }; \ if [ -f "$(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json" ]; then \ echo " πŸ“Š Verification: Target file exists and has $$(wc -c < "$(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json") bytes"; \ else \ echo " ❌ Verification failed: Target file does not exist"; exit 1; \ fi; \ else \ echo " ⚠️ Source file $(TOOL_ROOT)/gen/$(TAG)-properties.json not found, skipping copy"; \ fi @echo "🧹 Cleaning up old versioned JSON files (keeping only 2 most recent)…" @cd "$(OUTPUT_JSON_DIR)" && \ ls -t redpanda-properties-*.json 2>/dev/null | tail -n +3 | while read file; do \ echo " Removing old file: $$file"; \ rm -f "$$file"; \ done || true @echo "βœ… Docs generated at $(OUTPUT_AUTOGENERATED_DIR)" # --- Compare two property files and generate a diff report --- # Usage: make compare OLD_TAG=v25.2.1 NEW_TAG=v25.2.9 OVERRIDES=path/to/overrides.json # This will compare the two versions and save the diff to the overrides directory compare: @if [ -z "$(OLD_TAG)" ] || [ -z "$(NEW_TAG)" ]; then \ echo "❌ Error: Both OLD_TAG and NEW_TAG must be specified"; \ echo " Usage: make compare OLD_TAG=v25.2.1 NEW_TAG=v25.2.9 OVERRIDES=path/to/overrides.json"; \ exit 1; \ fi @if [ -z "$(OVERRIDES)" ]; then \ echo "❌ Error: OVERRIDES must be specified to determine where to save the diff"; \ echo " Usage: make compare OLD_TAG=v25.2.1 NEW_TAG=v25.2.9 OVERRIDES=path/to/overrides.json"; \ exit 1; \ fi @if [ ! -f "$(OUTPUT_JSON_DIR)/redpanda-properties-$(OLD_TAG).json" ]; then \ echo "❌ Error: Old properties file not found: $(OUTPUT_JSON_DIR)/redpanda-properties-$(OLD_TAG).json"; \ exit 1; \ fi @if [ ! -f "$(OUTPUT_JSON_DIR)/redpanda-properties-$(NEW_TAG).json" ]; then \ echo "❌ Error: New properties file not found: $(OUTPUT_JSON_DIR)/redpanda-properties-$(NEW_TAG).json"; \ exit 1; \ fi @echo "Comparing $(OLD_TAG) β†’ $(NEW_TAG)" @echo " Diff will be saved to: $(OUTPUT_DIFF_DIR)" @cd $(TOOL_ROOT) && \ node compare-properties.js \ "$(OUTPUT_JSON_DIR)/redpanda-properties-$(OLD_TAG).json" \ "$(OUTPUT_JSON_DIR)/redpanda-properties-$(NEW_TAG).json" \ $(OLD_TAG) \ $(NEW_TAG) \ "$(OUTPUT_DIFF_DIR)" \ "redpanda-property-changes-$(OLD_TAG)-to-$(NEW_TAG).json" @echo "βœ… Diff saved to $(OUTPUT_DIFF_DIR)/redpanda-property-changes-$(OLD_TAG)-to-$(NEW_TAG).json" @$(MAKE) cleanup-diffs OVERRIDES="$(OVERRIDES)" # --- Helper target to cleanup old diff files --- cleanup-diffs: @echo "🧹 Cleaning up old diff JSON files (keeping only 2 most recent)…" @cd "$(OUTPUT_DIFF_DIR)" && \ ls -t redpanda-property-changes-*.json 2>/dev/null | tail -n +3 | while read file; do \ echo " Removing old file: $$file"; \ rm -f "$$file"; \ done || true # --- Debug helper to print all the key paths/vars --- check: @echo "MODULE_ROOT: $(MODULE_ROOT)" @echo "TOOL_ROOT: $(TOOL_ROOT)" @echo "REDPANDA_SRC: $(REDPANDA_SRC)" @echo "TREESITTER: $(TREESITTER_DIR)" @echo "VENV: $(VENV)" @echo "PYTHON: $(PYTHON)" @echo "OVERRIDES: $(OVERRIDES)" @echo "OUTPUT_ASCIIDOC_DIR: $(OUTPUT_ASCIIDOC_DIR)" @echo "OUTPUT_JSON_DIR: $(OUTPUT_JSON_DIR)" @echo "OUTPUT_DIFF_DIR: $(OUTPUT_DIFF_DIR)" @echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"