Files

593 lines
23 KiB
Python

import importlib
import json
import os
import platform
import shutil
import subprocess
import sys
import sysconfig
from pathlib import Path
from setuptools import Distribution, Extension, setup
from setuptools.command.build_ext import build_ext
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError: # pragma: no cover - local build_ext may not have wheel installed
bdist_wheel = None
SETUP_DIR = Path(__file__).resolve().parent
if str(SETUP_DIR) not in sys.path:
sys.path.insert(0, str(SETUP_DIR))
get_host_engine_build_config = importlib.import_module(
"scripts.build_support.x86_profiles"
).get_host_engine_build_config
resolve_openviking_version = importlib.import_module(
"scripts.build_support.versioning"
).resolve_openviking_version
CMAKE_PATH = shutil.which("cmake") or "cmake"
C_COMPILER_PATH = os.environ.get("CC") or shutil.which("gcc") or "gcc"
CXX_COMPILER_PATH = os.environ.get("CXX") or shutil.which("g++") or "g++"
ENGINE_SOURCE_DIR = "src/"
ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())
SKIP_CPP_BUILD = os.environ.get("OV_SKIP_CPP_BUILD") == "1"
def _sanitize_native_build_env(env):
"""Keep Rust native builds from accidentally linking against Linuxbrew libs.
On older glibc systems, Homebrew-provided native libraries can require a newer
libc than the host linker/runtime supports. When pkg-config resolves xz/bzip2
from Linuxbrew, Cargo inherits those library search paths and link fails.
"""
sanitized_env = env.copy()
pkg_config = sanitized_env.get("PKG_CONFIG") or shutil.which("pkg-config")
if pkg_config and "linuxbrew" in os.path.realpath(pkg_config).lower():
system_pkg_config = "/usr/bin/pkg-config"
if Path(system_pkg_config).exists():
sanitized_env["PKG_CONFIG"] = system_pkg_config
for key in ("PKG_CONFIG_PATH", "LIBRARY_PATH", "LD_LIBRARY_PATH"):
value = sanitized_env.get(key)
if not value:
continue
kept_paths = [
path
for path in value.split(os.pathsep)
if path and "linuxbrew" not in os.path.realpath(path).lower()
]
if kept_paths:
sanitized_env[key] = os.pathsep.join(kept_paths)
else:
sanitized_env.pop(key, None)
return sanitized_env
def _get_windows_python_sabi_library() -> Path:
"""Return the stable-ABI Python library path for Windows abi3 extensions."""
candidate_roots = []
for raw_root in (
sys.base_prefix,
sys.base_exec_prefix,
sysconfig.get_config_var("installed_base"),
sysconfig.get_config_var("base"),
):
if not raw_root:
continue
candidate_root = Path(raw_root).resolve()
if candidate_root not in candidate_roots:
candidate_roots.append(candidate_root)
candidate_paths = []
for root in candidate_roots:
candidate_paths.extend(
[
root / "libs" / "python3.lib",
root / "python3.dll",
]
)
for candidate_path in candidate_paths:
if candidate_path.exists():
return candidate_path
searched = ", ".join(str(path) for path in candidate_paths) or "<none>"
raise RuntimeError(
"Could not locate the Windows stable-ABI Python library for abi3 engine modules. "
f"Searched: {searched}"
)
class OpenVikingBuildExt(build_ext):
"""Build OpenViking runtime artifacts and Python native extensions."""
def run(self):
self.build_ov_cli_artifact()
self.build_ragfs_python_artifact()
self.cmake_executable = CMAKE_PATH
if SKIP_CPP_BUILD:
print("[SKIP] C++ vector engine build disabled by OV_SKIP_CPP_BUILD=1")
else:
for ext in self.extensions:
self.build_extension(ext)
def _copy_artifact(self, src, dst):
"""Copy a build artifact into the package tree and preserve executability."""
print(f"Copying artifact from {src} to {dst}")
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(src), str(dst))
if sys.platform != "win32":
os.chmod(str(dst), 0o755)
def _copy_artifacts_to_build_lib(self, target_binary=None, target_lib=None):
"""Copy built artifacts into build_lib so wheel packaging can include them."""
if self.build_lib:
build_pkg_dir = Path(self.build_lib) / "openviking"
if target_binary and target_binary.exists():
self._copy_artifact(target_binary, build_pkg_dir / "bin" / target_binary.name)
if target_lib and target_lib.exists():
self._copy_artifact(target_lib, build_pkg_dir / "lib" / target_lib.name)
def _require_artifact(self, artifact_path, artifact_name, stage_name):
"""Abort the build immediately when a required artifact is missing."""
if artifact_path.exists():
return
raise RuntimeError(
f"{stage_name} did not produce required {artifact_name} at {artifact_path}"
)
def _run_stage_with_artifact_checks(
self, stage_name, build_fn, required_artifacts, on_success=None
):
"""Run a build stage and always validate its required outputs on normal return."""
build_fn()
for artifact_path, artifact_name in required_artifacts:
self._require_artifact(artifact_path, artifact_name, stage_name)
if on_success:
on_success()
def _resolve_cargo_target_dir(self, cargo_project_dir, env):
"""Resolve the Cargo target directory for workspace and overridden builds."""
configured_target_dir = env.get("CARGO_TARGET_DIR")
if configured_target_dir:
return Path(configured_target_dir).resolve()
try:
result = subprocess.run(
["cargo", "metadata", "--format-version", "1", "--no-deps"],
cwd=str(cargo_project_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
metadata = json.loads(result.stdout.decode("utf-8"))
target_directory = metadata.get("target_directory")
if target_directory:
return Path(target_directory).resolve()
except Exception as exc:
print(f"[Warning] Failed to resolve Cargo target directory via metadata: {exc}")
return cargo_project_dir.parents[1] / "target"
def build_ov_cli_artifact(self):
"""Build or reuse the ov Rust CLI binary."""
if os.environ.get("OV_SKIP_OV_BUILD") == "1":
print("[SKIP] ov CLI build disabled by OV_SKIP_OV_BUILD=1")
return
binary_name = "ov.exe" if sys.platform == "win32" else "ov"
ov_cli_dir = Path("crates/ov_cli").resolve()
ov_target_binary = Path("openviking/bin").resolve() / binary_name
self._run_stage_with_artifact_checks(
"ov CLI build",
lambda: self._build_ov_cli_artifact_impl(ov_cli_dir, binary_name, ov_target_binary),
[(ov_target_binary, binary_name)],
on_success=lambda: self._copy_artifacts_to_build_lib(ov_target_binary, None),
)
def _build_ov_cli_artifact_impl(self, ov_cli_dir, binary_name, ov_target_binary):
"""Implement ov CLI building without final artifact checks."""
prebuilt_dir = os.environ.get("OV_PREBUILT_BIN_DIR")
if prebuilt_dir:
src_bin = Path(prebuilt_dir).resolve() / binary_name
if src_bin.exists():
self._copy_artifact(src_bin, ov_target_binary)
return
if ov_cli_dir.exists() and shutil.which("cargo"):
print("Building ov CLI from source...")
try:
env = _sanitize_native_build_env(os.environ.copy())
env["OPENVIKING_VERSION"] = resolve_openviking_version(
env=env, project_root=SETUP_DIR
)
build_args = ["cargo", "build", "--release"]
target = env.get("CARGO_BUILD_TARGET")
if target:
print(f"Cross-compiling with CARGO_BUILD_TARGET={target}")
build_args.extend(["--target", target])
result = subprocess.run(
build_args,
cwd=str(ov_cli_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.stdout:
print(f"Build stdout: {result.stdout.decode('utf-8', errors='replace')}")
if result.stderr:
print(f"Build stderr: {result.stderr.decode('utf-8', errors='replace')}")
cargo_target_dir = self._resolve_cargo_target_dir(ov_cli_dir, env)
if target:
built_bin = cargo_target_dir / target / "release" / binary_name
else:
built_bin = cargo_target_dir / "release" / binary_name
self._require_artifact(built_bin, binary_name, "ov CLI build")
self._copy_artifact(built_bin, ov_target_binary)
print("[OK] ov CLI built successfully from source")
except Exception as exc:
error_msg = f"Failed to build ov CLI from source: {exc}"
if isinstance(exc, subprocess.CalledProcessError):
if exc.stdout:
error_msg += (
f"\nBuild stdout:\n{exc.stdout.decode('utf-8', errors='replace')}"
)
if exc.stderr:
error_msg += (
f"\nBuild stderr:\n{exc.stderr.decode('utf-8', errors='replace')}"
)
print(f"[Error] {error_msg}")
raise RuntimeError(error_msg)
else:
if ov_target_binary.exists():
print("[Info] ov CLI binary already exists locally. Skipping source build.")
elif not ov_cli_dir.exists():
print(f"[Warning] ov CLI source directory not found at {ov_cli_dir}")
else:
print("[Warning] Cargo not found. Cannot build ov CLI from source.")
def build_ragfs_python_artifact(self):
"""Build ragfs-python (Rust RAGFS binding) via maturin and copy the native
extension into ``openviking/lib/`` so it ships inside the openviking wheel.
"""
require_ragfs_artifact = self._should_require_ragfs_artifact()
ragfs_python_dir = Path("crates/ragfs-python").resolve()
ragfs_lib_dir = Path("openviking/lib").resolve()
if not ragfs_python_dir.exists():
message = "ragfs-python source directory not found."
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[Info] {message} Skipping.")
return
if os.environ.get("OV_SKIP_RAGFS_BUILD") == "1":
message = "Skipping ragfs-python build (OV_SKIP_RAGFS_BUILD=1)"
if require_ragfs_artifact:
raise RuntimeError(f"{message} is incompatible with required wheel artifacts.")
print(f"[OK] {message}")
return
if importlib.util.find_spec("maturin") is None:
message = (
"maturin not found. ragfs-python (Rust binding) will not be built.\n"
" Install maturin to enable: pip install maturin"
)
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[SKIP] {message}")
return
import tempfile
import zipfile
with tempfile.TemporaryDirectory() as tmpdir:
try:
print("Building ragfs-python (Rust RAGFS binding) via maturin...")
env = _sanitize_native_build_env(os.environ.copy())
build_args = [
sys.executable,
"-m",
"maturin",
"build",
"--release",
"--out",
tmpdir,
]
# Respect CARGO_BUILD_TARGET for cross-compilation
target = env.get("CARGO_BUILD_TARGET")
if target:
build_args.extend(["--target", target])
result = subprocess.run(
build_args,
cwd=str(ragfs_python_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.stdout:
print(result.stdout.decode("utf-8", errors="replace"))
if result.stderr:
print(result.stderr.decode("utf-8", errors="replace"))
# Extract the stable-ABI native extension from the built wheel.
whl_files = list(Path(tmpdir).glob("ragfs_python-*.whl"))
if not whl_files:
message = "maturin produced no wheel for ragfs-python."
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[Warning] {message}")
return
ragfs_lib_dir.mkdir(parents=True, exist_ok=True)
for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.so"):
stale_artifact.unlink()
for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.pyd"):
stale_artifact.unlink()
for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.dylib"):
stale_artifact.unlink()
extracted = False
with zipfile.ZipFile(str(whl_files[0])) as zf:
for name in zf.namelist():
basename = Path(name).name
is_ragfs_extension = basename == "ragfs_python.pyd" or (
basename.startswith("ragfs_python.abi3.")
and basename.endswith((".so", ".pyd"))
)
if is_ragfs_extension:
target_path = ragfs_lib_dir / basename
with zf.open(name) as src, open(target_path, "wb") as dst:
dst.write(src.read())
if sys.platform != "win32":
os.chmod(str(target_path), 0o755)
print(f"[OK] ragfs-python: extracted {basename} -> {target_path}")
extracted = True
break
if not extracted:
message = (
"Could not find ragfs_python stable-ABI native extension in built wheel."
)
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[Warning] {message}")
else:
self._copy_artifacts_to_build_lib(target_lib=target_path)
except Exception as exc:
error_detail = ""
if isinstance(exc, subprocess.CalledProcessError):
if exc.stdout:
error_detail += exc.stdout.decode("utf-8", errors="replace")
if exc.stderr:
error_detail += exc.stderr.decode("utf-8", errors="replace")
if require_ragfs_artifact:
error_message = f"Failed to build ragfs-python: {exc}"
if error_detail:
error_message += f"\n{error_detail}"
raise RuntimeError(error_message) from exc
print(f"[Warning] Failed to build ragfs-python: {exc}")
if error_detail:
print(error_detail)
def _should_require_ragfs_artifact(self) -> bool:
"""Fail wheel builds closed when ragfs-python cannot be bundled."""
required = os.environ.get("OV_REQUIRE_RAGFS_BUILD")
if required is not None:
return required == "1"
return "bdist_wheel" in sys.argv
def build_extension(self, ext):
"""Build a single Python native extension artifact using CMake."""
if getattr(self, "_engine_extensions_built", False):
return
ext_fullpath = Path(self.get_ext_fullpath(ext.name))
ext_dir = ext_fullpath.parent.resolve()
build_dir = Path(self.build_temp) / "cmake_build"
build_dir.mkdir(parents=True, exist_ok=True)
self._clean_stale_engine_artifacts(ext_dir)
self._run_stage_with_artifact_checks(
"CMake build",
lambda: self._build_extension_impl(ext_fullpath, ext_dir, build_dir),
[(ext_fullpath, f"native extension '{ext.name}'")],
)
self._engine_extensions_built = True
def _clean_stale_engine_artifacts(self, ext_dir: Path):
"""Remove stale non-abi3 engine binaries from wheel build output directories."""
source_engine_dir = (SETUP_DIR / "openviking" / "storage" / "vectordb" / "engine").resolve()
if ext_dir == source_engine_dir:
return
for pattern in ("*.so", "*.pyd"):
for artifact in ext_dir.glob(pattern):
artifact.unlink()
def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
"""Invoke CMake to build the Python native extension."""
ext_basename = ext_fullpath.stem.split(".")[0]
built_filename = Path(self.get_ext_filename(self.extensions[0].name)).name
py_ext_suffix = built_filename.removeprefix(ext_basename)
if not py_ext_suffix:
py_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ext_fullpath.suffix
cmake_args = [
f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",
f"-B{build_dir}",
"-DCMAKE_BUILD_TYPE=Release",
f"-DOV_PY_OUTPUT_DIR={ext_dir}",
f"-DOV_PY_EXT_SUFFIX={py_ext_suffix}",
f"-DOV_X86_BUILD_VARIANTS={';'.join(ENGINE_BUILD_CONFIG.cmake_variants)}",
"-DCMAKE_VERBOSE_MAKEFILE=ON",
"-DCMAKE_INSTALL_RPATH=$ORIGIN",
f"-DPython3_EXECUTABLE={sys.executable}",
f"-DPython3_INCLUDE_DIRS={sysconfig.get_path('include')}",
f"-DPython3_LIBRARIES={sysconfig.get_config_vars().get('LIBRARY')}",
f"-DCMAKE_C_COMPILER={C_COMPILER_PATH}",
f"-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}",
]
if sys.platform == "darwin":
cmake_args.append("-DCMAKE_OSX_DEPLOYMENT_TARGET=10.15")
target_arch = os.environ.get("CMAKE_OSX_ARCHITECTURES")
if target_arch:
cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}")
elif sys.platform == "win32":
windows_python_sabi_library = _get_windows_python_sabi_library()
cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}")
cmake_args.extend(["-G", "MinGW Makefiles"])
self.spawn([self.cmake_executable] + cmake_args)
build_args = ["--build", str(build_dir), "--config", "Release", f"-j{os.cpu_count() or 4}"]
self.spawn([self.cmake_executable] + build_args)
def _build_web_studio():
"""Build the web-studio SPA and copy dist into the Python package tree.
Skipped when OV_SKIP_STUDIO_BUILD=1 or when the bundle already exists.
Falls back gracefully (warning, not error) when npm is unavailable.
"""
if os.environ.get("OV_SKIP_STUDIO_BUILD") == "1":
print(" [SKIP] web-studio build disabled by OV_SKIP_STUDIO_BUILD=1")
return
dest = SETUP_DIR / "openviking" / "web_studio" / "dist"
if (dest / "index.html").is_file():
print(" [OK] web-studio bundle already present")
return
source = SETUP_DIR / "web-studio"
if not (source / "package.json").is_file():
print(" [SKIP] web-studio source not found; /studio will be unavailable")
return
npm = shutil.which("npm")
if not npm:
print(" [SKIP] npm not found; install Node.js to enable /studio")
return
print("Building web-studio (Vite SPA)...")
try:
subprocess.check_call([npm, "ci"], cwd=str(source))
subprocess.check_call(
[npm, "run", "build", "--", "--base=/studio/"],
cwd=str(source),
)
except subprocess.CalledProcessError as exc:
print(f" [WARNING] web-studio npm build failed ({exc}); /studio will be unavailable")
return
built = source / "dist"
if not (built / "index.html").is_file():
print(" [WARNING] web-studio build produced no index.html; /studio will be unavailable")
return
dest.parent.mkdir(parents=True, exist_ok=True)
if dest.exists():
shutil.rmtree(dest)
shutil.copytree(built, dest)
print(f" [OK] web-studio bundle copied to {dest}")
class OpenVikingBuildPy(build_py):
def run(self):
_build_web_studio()
super().run()
package_root = Path(self.build_lib) / "vikingbot"
for asset_name in ("workspace", "bridge"):
source = SETUP_DIR / "bot" / asset_name
if not source.is_dir():
raise FileNotFoundError(
f"required VikingBot asset directory not found: {source}"
)
destination = package_root / asset_name
if destination.exists():
shutil.rmtree(destination)
shutil.copytree(
source,
destination,
ignore=shutil.ignore_patterns(
"node_modules", "dist", "__pycache__", "*.pyc"
),
)
if bdist_wheel is not None:
class OpenVikingBdistWheel(bdist_wheel):
def finalize_options(self):
super().finalize_options()
self.py_limited_api = "cp310"
else:
OpenVikingBdistWheel = None
class OpenVikingDistribution(Distribution):
"""Keep wheels platform-specific because ragfs-python is a native artifact."""
def has_ext_modules(self):
return True
cmdclass = {
"build_ext": OpenVikingBuildExt,
"build_py": OpenVikingBuildPy,
}
if OpenVikingBdistWheel is not None:
cmdclass["bdist_wheel"] = OpenVikingBdistWheel
setup(
distclass=OpenVikingDistribution,
ext_modules=(
[]
if SKIP_CPP_BUILD
else [
Extension(
name=ENGINE_BUILD_CONFIG.primary_extension,
sources=[],
py_limited_api=True,
)
]
),
cmdclass=cmdclass,
package_data={
"openviking": [
"lib/ragfs_python*.so",
"lib/ragfs_python*.pyd",
"bin/ov",
"bin/ov.exe",
"server/static/**/*",
"web_studio/dist/**/*",
"storage/vectordb/engine/*.abi3.so",
"storage/vectordb/engine/*.pyd",
],
"vikingbot": [
"workspace/**/*",
"bridge/**/*",
],
},
include_package_data=True,
)