diff --git a/ci/official/any.sh b/ci/official/any.sh index 8bfba811d74..0548e4e9e1b 100755 --- a/ci/official/any.sh +++ b/ci/official/any.sh @@ -7,15 +7,17 @@ source "${BASH_SOURCE%/*}/utilities/setup.sh" # subshell environment. # Ignore grep failures since we're using it for basic filtering set +e -filtered_build_targets=( $(echo "$BUILD_TARGETS" | tr ' ' '\n' | grep . | tee build_targets.txt) ) -nonpip_targets=( $(echo "$TEST_TARGETS" | tr ' ' '\n' | grep -E "^//tensorflow/" | tee nonpip_targets.txt) ) +filtered_build_targets=( $(echo "$BUILD_TARGETS" | tr ' ' '\n' | grep .) ) +nonpip_targets=( $(echo "$TEST_TARGETS" | tr ' ' '\n' | grep -E "^//tensorflow/" ) ) config=( $(echo "$CONFIG_OPTIONS" ) ) test_flags=( $(echo "$TEST_FLAGS" ) ) set -e -[[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]] && tfrun nvidia-smi +if [[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]]; then + tfrun nvidia-smi +fi -if [[ -s build_targets.txt ]]; then +if [[ "${#filtered_build_targets[@]}" -ne 0 ]]; then tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" "${config[@]}" "${filtered_build_targets[@]}" fi @@ -23,11 +25,11 @@ if [[ "${PIP_WHEEL}" -eq "1" ]]; then # Update the version numbers to build a "nightly" package [[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]] && tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly - tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_CACHE_ARGS[@]}" tensorflow/tools/pip_package:build_pip_package + tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_COMMON_ARGS[@]}" tensorflow/tools/pip_package:build_pip_package tfrun ./bazel-bin/tensorflow/tools/pip_package/build_pip_package build "${TFCI_BUILD_PIP_PACKAGE_ARGS[@]}" tfrun ./ci/official/utilities/rename_and_verify_wheels.sh fi -if [[ -s nonpip_targets.txt ]]; then +if [[ "${#nonpip_targets[@]}" -ne 0 ]]; then tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${config[@]}" "${test_flags[@]}" "${nonpip_targets[@]}" fi diff --git a/ci/official/bazelrcs/cpu.bazelrc b/ci/official/bazelrcs/cpu.bazelrc index 3a324603bdf..2a1597e24a5 100644 --- a/ci/official/bazelrcs/cpu.bazelrc +++ b/ci/official/bazelrcs/cpu.bazelrc @@ -37,7 +37,7 @@ build --copt=-Wno-gnu-offsetof-extensions # Store performance profiling log in the mounted artifact directory. # The profile can be viewed by visiting chrome://tracing in a Chrome browser. # See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling -build --profile=/tf/pkg/profile.json.gz +build --profile=build/profile.json.gz # Use the NVCC toolchain to compile for manylinux2014 build --crosstool_top="@sigbuild-r2.14-clang_config_cuda//crosstool:toolchain" @@ -63,7 +63,7 @@ test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //t build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip # For outputting Build Event Protocol files -build:build_event_export --build_event_json_file=/tf/pkg/bep.json +build:build_event_export --build_event_json_file=build/bep.json # For Remote Build Execution. build:rbe --google_default_credentials diff --git a/ci/official/bazelrcs/cpu_gcc.bazelrc b/ci/official/bazelrcs/cpu_gcc.bazelrc index cc74fd978cf..311d59b27a0 100644 --- a/ci/official/bazelrcs/cpu_gcc.bazelrc +++ b/ci/official/bazelrcs/cpu_gcc.bazelrc @@ -26,7 +26,7 @@ build --copt=-mavx --host_copt=-mavx # Store performance profiling log in the mounted artifact directory. # The profile can be viewed by visiting chrome://tracing in a Chrome browser. # See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling -build --profile=/tf/pkg/profile.json.gz +build --profile=build/profile.json.gz # Use the NVCC toolchain to compile for manylinux2014 build --crosstool_top="@sigbuild-r2.14_config_cuda//crosstool:toolchain" @@ -52,7 +52,7 @@ test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //t build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip # For outputting Build Event Protocol files -build:build_event_export --build_event_json_file=/tf/pkg/bep.json +build:build_event_export --build_event_json_file=build/bep.json # For Remote Build Execution. build:rbe --google_default_credentials diff --git a/ci/official/bazelrcs/nvidia.bazelrc b/ci/official/bazelrcs/nvidia.bazelrc index 50ea5752059..3a6773579e9 100644 --- a/ci/official/bazelrcs/nvidia.bazelrc +++ b/ci/official/bazelrcs/nvidia.bazelrc @@ -37,7 +37,7 @@ build --linkopt="-lm" # Store performance profiling log in the mounted artifact directory. # The profile can be viewed by visiting chrome://tracing in a Chrome browser. # See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling -build --profile=/tf/pkg/profile.json.gz +build --profile=build/profile.json.gz # CUDA: Set up compilation CUDA version and paths build --@local_config_cuda//:enable_cuda @@ -86,7 +86,7 @@ test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //t build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip # For outputting Build Event Protocol files -build:build_event_export --build_event_json_file=/tf/pkg/bep.json +build:build_event_export --build_event_json_file=build/bep.json # For Remote Build Execution. build:rbe --google_default_credentials diff --git a/ci/official/envs/local_cpu b/ci/official/envs/local_cpu index 914dd4c856a..79e64bfc28b 100644 --- a/ci/official/envs/local_cpu +++ b/ci/official/envs/local_cpu @@ -1,12 +1,12 @@ TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache) TFCI_BUILD_PIP_PACKAGE_ARGS=("--cpu") TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 TFCI_DOCKER_GPU_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9 TFCI_DOCKER_PULL_ENABLE= -TFCI_GIT_DIR=/usr/local/google/home/angerson/repos/tensorflow +TFCI_GIT_DIR=. TFCI_INDEX_HTML_ENABLE=1 TFCI_LIB_SUFFIX="-cpu-linux-x86_64" TFCI_NIGHTLY_UPDATE_VERSION_ENABLE= diff --git a/ci/official/envs/nightly_cpu_py310 b/ci/official/envs/nightly_cpu_py310 index 9cff1f3803f..eabe2dcc845 100644 --- a/ci/official/envs/nightly_cpu_py310 +++ b/ci/official/envs/nightly_cpu_py310 @@ -1,7 +1,7 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=1 TFCI_DOCKER_ENABLE=1 diff --git a/ci/official/envs/nightly_cpu_py311 b/ci/official/envs/nightly_cpu_py311 index e28e8f6cf3c..0201e5aa44c 100644 --- a/ci/official/envs/nightly_cpu_py311 +++ b/ci/official/envs/nightly_cpu_py311 @@ -1,7 +1,7 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=1 TFCI_DOCKER_ENABLE=1 diff --git a/ci/official/envs/nightly_cpu_py39 b/ci/official/envs/nightly_cpu_py39 index 6c34a60b89c..436bd41e169 100644 --- a/ci/official/envs/nightly_cpu_py39 +++ b/ci/official/envs/nightly_cpu_py39 @@ -1,7 +1,7 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=1 TFCI_DOCKER_ENABLE=1 diff --git a/ci/official/envs/nightly_nvidia_py310 b/ci/official/envs/nightly_nvidia_py310 index dbfd3ca756b..214efe40d42 100644 --- a/ci/official/envs/nightly_nvidia_py310 +++ b/ci/official/envs/nightly_nvidia_py310 @@ -1,7 +1,7 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/nvidia.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push) TFCI_BUILD_PIP_PACKAGE_ARGS=(--gpu --nightly_flag) TFCI_COPYBARA_ENABLE=1 TFCI_DOCKER_ENABLE=1 diff --git a/ci/official/envs/nightly_nvidia_py311 b/ci/official/envs/nightly_nvidia_py311 index c988660c54e..9a4a8f173eb 100644 --- a/ci/official/envs/nightly_nvidia_py311 +++ b/ci/official/envs/nightly_nvidia_py311 @@ -1,7 +1,7 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/nvidia.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push) TFCI_BUILD_PIP_PACKAGE_ARGS=(--gpu --nightly_flag) TFCI_COPYBARA_ENABLE=1 TFCI_DOCKER_ENABLE=1 diff --git a/ci/official/envs/nightly_nvidia_py39 b/ci/official/envs/nightly_nvidia_py39 index 9c0984080a2..4e729536b1d 100644 --- a/ci/official/envs/nightly_nvidia_py39 +++ b/ci/official/envs/nightly_nvidia_py39 @@ -1,7 +1,7 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/nvidia.bazelrc) -TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push) TFCI_BUILD_PIP_PACKAGE_ARGS=(--gpu --nightly_flag) TFCI_COPYBARA_ENABLE=1 TFCI_DOCKER_ENABLE=1 diff --git a/ci/official/libtensorflow.sh b/ci/official/libtensorflow.sh index 31ed771f40b..db3647bec7d 100755 --- a/ci/official/libtensorflow.sh +++ b/ci/official/libtensorflow.sh @@ -2,13 +2,17 @@ source "${BASH_SOURCE%/*}/utilities/setup.sh" # Record GPU count and CUDA version status -[[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]] && tfrun nvidia-smi +if [[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]]; then + tfrun nvidia-smi +fi # Update the version numbers for Nightly only -[[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]] && tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly +if [[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]]; then + tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly +fi -tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=libtensorflow_test -tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=libtensorflow_build +tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=libtensorflow_test +tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=libtensorflow_build tfrun ./ci/official/utilities/repack_libtensorflow.sh build "$TFCI_LIB_SUFFIX" diff --git a/ci/official/pycpp.sh b/ci/official/pycpp.sh index 984e27d021a..f6aef18b96c 100755 --- a/ci/official/pycpp.sh +++ b/ci/official/pycpp.sh @@ -1,6 +1,6 @@ #!/bin/bash source "${BASH_SOURCE%/*}/utilities/setup.sh" -tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=rbe --config=pycpp --config=build_event_export +tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=pycpp tfrun bazel analyze-profile build/profile.json.gz diff --git a/ci/official/utilities/copybara.sh b/ci/official/utilities/copybara.sh index 34f24c802fd..fd9dac640d2 100755 --- a/ci/official/utilities/copybara.sh +++ b/ci/official/utilities/copybara.sh @@ -1,11 +1,4 @@ #!/bin/bash -# -e: abort script if one command fails -# -u: error if undefined variable used -# -o pipefail: entire command fails if pipe fails. watch out for yes | ... -# -o history: record shell history -set -euxo pipefail -o history -set -o allexport && source "$TFCI" && set +o allexport - # Destroy any existing github code rm -rf "$TFCI_GIT_DIR" mkdir -p "$TFCI_GIT_DIR" diff --git a/ci/official/utilities/docker.sh b/ci/official/utilities/docker.sh index b84ee381e51..e0168a4b94b 100755 --- a/ci/official/utilities/docker.sh +++ b/ci/official/utilities/docker.sh @@ -1,11 +1,4 @@ #!/bin/bash -# -e: abort script if one command fails -# -u: error if undefined variable used -# -o pipefail: entire command fails if pipe fails. watch out for yes | ... -# -o history: record shell history -set -euxo pipefail -o history -set -o allexport && source "$TFCI" && set +o allexport - trap "docker rm -f tf" EXIT if [[ "$TFCI_DOCKER_PULL_ENABLE" == 1 ]]; then docker pull "$TFCI_DOCKER_IMAGE" diff --git a/ci/official/utilities/repack_libtensorflow.sh b/ci/official/utilities/repack_libtensorflow.sh index fefce92f747..7492642148f 100755 --- a/ci/official/utilities/repack_libtensorflow.sh +++ b/ci/official/utilities/repack_libtensorflow.sh @@ -19,13 +19,6 @@ # # Repacks libtensorflow tarballs into $DIR with provided $TARBALL_SUFFIX, # and also repacks libtensorflow-src.jar into a standardized format. -# -# -e: abort script if one command fails -# -u: error if undefined variable used -# -o pipefail: entire command fails if pipe fails. watch out for yes | ... -# -o history: record shell history -set -euxo pipefail -o history -set -o allexport && source "$TFCI" && set +o allexport # Helper function to copy a srcjar after moving any source files # directly under the root to the "maven-style" src/main/java layout diff --git a/ci/official/utilities/setup.sh b/ci/official/utilities/setup.sh index ba1988964b0..3d5773ffc32 100755 --- a/ci/official/utilities/setup.sh +++ b/ci/official/utilities/setup.sh @@ -13,15 +13,19 @@ # -o pipefail: entire command fails if pipe fails. watch out for yes | ... # -o history: record shell history # -o allexport: export all functions and variables to be available to subscripts +# (affects 'source $TFCI') set -euxo pipefail -o history -o allexport # Import all variables as set in $TFCI, which should be a file like those in # the envs directory that sets all TFCI_ variables, e.g. /path/to/envs/local_cpu -source "$TFCI" +if [[ -n "$TFCI" ]]; then + source "$TFCI" +fi # Make a "build" directory for outputting all build artifacts (TF's .gitignore # ignores the "build" directory) -cd "$TFCI_GIT_DIR" && mkdir -p build +cd "$TFCI_GIT_DIR" +mkdir -p build # Setup tfrun, a helper function for executing steps that can either be run # locally or run under Docker. docker.sh, below, redefines it as "docker exec". @@ -34,10 +38,16 @@ tfrun() { "$@"; } # For Google-internal jobs, run copybara, which will overwrite the source tree. # Never useful for outside users. -[[ "$TFCI_COPYBARA_ENABLE" == 1 ]] && source ./ci/official/utilities/copybara.sh +if [[ "$TFCI_COPYBARA_ENABLE" == 1 ]]; then + source ./ci/official/utilities/copybara.sh +fi # Run all "tfrun" commands under Docker. See docker.sh for details -[[ "$TFCI_DOCKER_ENABLE" == 1 ]] && source ./ci/official/utilities/docker.sh +if [[ "$TFCI_DOCKER_ENABLE" == 1 ]]; then + source ./ci/official/utilities/docker.sh +fi # Generate an overview page describing the build -[[ "$TFCI_INDEX_HTML_ENABLE" == 1 ]] && ./ci/official/utilities/generate_index_html.sh build/index.html +if [[ "$TFCI_INDEX_HTML_ENABLE" == 1 ]]; then + ./ci/official/utilities/generate_index_html.sh build/index.html +fi diff --git a/ci/official/wheel.sh b/ci/official/wheel.sh index 359517a4052..81a7c260d5d 100755 --- a/ci/official/wheel.sh +++ b/ci/official/wheel.sh @@ -2,12 +2,16 @@ source "${BASH_SOURCE%/*}/utilities/setup.sh" # Record GPU count and CUDA version status -[[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]] && tfrun nvidia-smi +if [[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]]; then + tfrun nvidia-smi +fi # Update the version numbers for Nightly only -[[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]] && tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly +if [[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]]; then + tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly +fi -tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_CACHE_ARGS[@]}" //tensorflow/tools/pip_package:build_pip_package +tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_COMMON_ARGS[@]}" //tensorflow/tools/pip_package:build_pip_package tfrun ./bazel-bin/tensorflow/tools/pip_package/build_pip_package build "${TFCI_BUILD_PIP_PACKAGE_ARGS[@]}" tfrun ./ci/official/utilities/rename_and_verify_wheels.sh build @@ -16,4 +20,4 @@ if [[ "$TFCI_UPLOAD_ENABLE" == 1 ]]; then gsutil cp build/*.whl "$TFCI_UPLOAD_GCS_DESTINATION" fi -tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=nonpip +tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=nonpip