zzos/tensorflow
1
0
Fork 0
mirror of https://github.com/tensorflow/tensorflow.git synced 2026-09-28 13:23:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Handle feedback

Browse Source
This commit is contained in:
Austin Anderson
2023-07-07 17:31:00 -07:00
parent 5a85a6859d
commit 9c57db4e22
18 changed files with 54 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/official/any.sh
+8 -6
View File
@@ -7,15 +7,17 @@ source "${BASH_SOURCE%/*}/utilities/setup.sh"
# subshell environment.
# Ignore grep failures since we're using it for basic filtering
set +e
filtered_build_targets=( $(echo "$BUILD_TARGETS" | tr ' ' '\n' | grep . | tee build_targets.txt) )
nonpip_targets=( $(echo "$TEST_TARGETS" | tr ' ' '\n' | grep -E "^//tensorflow/" | tee nonpip_targets.txt) )
filtered_build_targets=( $(echo "$BUILD_TARGETS" | tr ' ' '\n' | grep .) )
nonpip_targets=( $(echo "$TEST_TARGETS" | tr ' ' '\n' | grep -E "^//tensorflow/" ) )
config=( $(echo "$CONFIG_OPTIONS" ) )
test_flags=( $(echo "$TEST_FLAGS" ) )
set -e
[[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]] && tfrun nvidia-smi
if [[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]]; then
tfrun nvidia-smi
fi
if [[ -s build_targets.txt ]]; then
if [[ "${#filtered_build_targets[@]}" -ne 0 ]]; then
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" "${config[@]}" "${filtered_build_targets[@]}"
fi
@@ -23,11 +25,11 @@ if [[ "${PIP_WHEEL}" -eq "1" ]]; then
# Update the version numbers to build a "nightly" package
[[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]] && tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_CACHE_ARGS[@]}" tensorflow/tools/pip_package:build_pip_package
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_COMMON_ARGS[@]}" tensorflow/tools/pip_package:build_pip_package
tfrun ./bazel-bin/tensorflow/tools/pip_package/build_pip_package build "${TFCI_BUILD_PIP_PACKAGE_ARGS[@]}"
tfrun ./ci/official/utilities/rename_and_verify_wheels.sh
fi
if [[ -s nonpip_targets.txt ]]; then
if [[ "${#nonpip_targets[@]}" -ne 0 ]]; then
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${config[@]}" "${test_flags[@]}" "${nonpip_targets[@]}"
fi
ci/official/bazelrcs/cpu.bazelrc
+2 -2
View File
@@ -37,7 +37,7 @@ build --copt=-Wno-gnu-offsetof-extensions
# Store performance profiling log in the mounted artifact directory.
# The profile can be viewed by visiting chrome://tracing in a Chrome browser.
# See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
build --profile=/tf/pkg/profile.json.gz
build --profile=build/profile.json.gz
# Use the NVCC toolchain to compile for manylinux2014
build --crosstool_top="@sigbuild-r2.14-clang_config_cuda//crosstool:toolchain"
@@ -63,7 +63,7 @@ test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //t
build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=/tf/pkg/bep.json
build:build_event_export --build_event_json_file=build/bep.json
# For Remote Build Execution.
build:rbe --google_default_credentials
ci/official/bazelrcs/cpu_gcc.bazelrc
+2 -2
View File
@@ -26,7 +26,7 @@ build --copt=-mavx --host_copt=-mavx
# Store performance profiling log in the mounted artifact directory.
# The profile can be viewed by visiting chrome://tracing in a Chrome browser.
# See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
build --profile=/tf/pkg/profile.json.gz
build --profile=build/profile.json.gz
# Use the NVCC toolchain to compile for manylinux2014
build --crosstool_top="@sigbuild-r2.14_config_cuda//crosstool:toolchain"
@@ -52,7 +52,7 @@ test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //t
build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=/tf/pkg/bep.json
build:build_event_export --build_event_json_file=build/bep.json
# For Remote Build Execution.
build:rbe --google_default_credentials
ci/official/bazelrcs/nvidia.bazelrc
+2 -2
View File
@@ -37,7 +37,7 @@ build --linkopt="-lm"
# Store performance profiling log in the mounted artifact directory.
# The profile can be viewed by visiting chrome://tracing in a Chrome browser.
# See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
build --profile=/tf/pkg/profile.json.gz
build --profile=build/profile.json.gz
# CUDA: Set up compilation CUDA version and paths
build --@local_config_cuda//:enable_cuda
@@ -86,7 +86,7 @@ test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //t
build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=/tf/pkg/bep.json
build:build_event_export --build_event_json_file=build/bep.json
# For Remote Build Execution.
build:rbe --google_default_credentials
ci/official/envs/local_cpu
+2 -2
View File
@@ -1,12 +1,12 @@
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache)
TFCI_BUILD_PIP_PACKAGE_ARGS=("--cpu")
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9
TFCI_DOCKER_PULL_ENABLE=
TFCI_GIT_DIR=/usr/local/google/home/angerson/repos/tensorflow
TFCI_GIT_DIR=.
TFCI_INDEX_HTML_ENABLE=1
TFCI_LIB_SUFFIX="-cpu-linux-x86_64"
TFCI_NIGHTLY_UPDATE_VERSION_ENABLE=
ci/official/envs/nightly_cpu_py310
+1 -1
View File
@@ -1,7 +1,7 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=1
TFCI_DOCKER_ENABLE=1
ci/official/envs/nightly_cpu_py311
+1 -1
View File
@@ -1,7 +1,7 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=1
TFCI_DOCKER_ENABLE=1
ci/official/envs/nightly_cpu_py39
+1 -1
View File
@@ -1,7 +1,7 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=1
TFCI_DOCKER_ENABLE=1
ci/official/envs/nightly_nvidia_py310
+1 -1
View File
@@ -1,7 +1,7 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/nvidia.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--gpu --nightly_flag)
TFCI_COPYBARA_ENABLE=1
TFCI_DOCKER_ENABLE=1
ci/official/envs/nightly_nvidia_py311
+1 -1
View File
@@ -1,7 +1,7 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/nvidia.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--gpu --nightly_flag)
TFCI_COPYBARA_ENABLE=1
TFCI_DOCKER_ENABLE=1
ci/official/envs/nightly_nvidia_py39
+1 -1
View File
@@ -1,7 +1,7 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/nvidia.bazelrc)
TFCI_BAZEL_CACHE_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--gpu --nightly_flag)
TFCI_COPYBARA_ENABLE=1
TFCI_DOCKER_ENABLE=1
ci/official/libtensorflow.sh
+8 -4
View File
@@ -2,13 +2,17 @@
source "${BASH_SOURCE%/*}/utilities/setup.sh"
# Record GPU count and CUDA version status
[[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]] && tfrun nvidia-smi
if [[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]]; then
tfrun nvidia-smi
fi
# Update the version numbers for Nightly only
[[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]] && tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly
if [[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]]; then
tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly
fi
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=libtensorflow_test
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=libtensorflow_build
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=libtensorflow_test
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=libtensorflow_build
tfrun ./ci/official/utilities/repack_libtensorflow.sh build "$TFCI_LIB_SUFFIX"
ci/official/pycpp.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
source "${BASH_SOURCE%/*}/utilities/setup.sh"
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=rbe --config=pycpp --config=build_event_export
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=pycpp
tfrun bazel analyze-profile build/profile.json.gz
ci/official/utilities/copybara.sh
-7
View File
@@ -1,11 +1,4 @@
#!/bin/bash
# -e: abort script if one command fails
# -u: error if undefined variable used
# -o pipefail: entire command fails if pipe fails. watch out for yes | ...
# -o history: record shell history
set -euxo pipefail -o history
set -o allexport && source "$TFCI" && set +o allexport
# Destroy any existing github code
rm -rf "$TFCI_GIT_DIR"
mkdir -p "$TFCI_GIT_DIR"
ci/official/utilities/docker.sh
-7
View File
@@ -1,11 +1,4 @@
#!/bin/bash
# -e: abort script if one command fails
# -u: error if undefined variable used
# -o pipefail: entire command fails if pipe fails. watch out for yes | ...
# -o history: record shell history
set -euxo pipefail -o history
set -o allexport && source "$TFCI" && set +o allexport
trap "docker rm -f tf" EXIT
if [[ "$TFCI_DOCKER_PULL_ENABLE" == 1 ]]; then
docker pull "$TFCI_DOCKER_IMAGE"
ci/official/utilities/repack_libtensorflow.sh
-7
View File
@@ -19,13 +19,6 @@
#
# Repacks libtensorflow tarballs into $DIR with provided $TARBALL_SUFFIX,
# and also repacks libtensorflow-src.jar into a standardized format.
#
# -e: abort script if one command fails
# -u: error if undefined variable used
# -o pipefail: entire command fails if pipe fails. watch out for yes | ...
# -o history: record shell history
set -euxo pipefail -o history
set -o allexport && source "$TFCI" && set +o allexport
# Helper function to copy a srcjar after moving any source files
# directly under the root to the "maven-style" src/main/java layout
ci/official/utilities/setup.sh
+15 -5
View File
@@ -13,15 +13,19 @@
# -o pipefail: entire command fails if pipe fails. watch out for yes | ...
# -o history: record shell history
# -o allexport: export all functions and variables to be available to subscripts
# (affects 'source $TFCI')
set -euxo pipefail -o history -o allexport
# Import all variables as set in $TFCI, which should be a file like those in
# the envs directory that sets all TFCI_ variables, e.g. /path/to/envs/local_cpu
source "$TFCI"
if [[ -n "$TFCI" ]]; then
source "$TFCI"
fi
# Make a "build" directory for outputting all build artifacts (TF's .gitignore
# ignores the "build" directory)
cd "$TFCI_GIT_DIR" && mkdir -p build
cd "$TFCI_GIT_DIR"
mkdir -p build
# Setup tfrun, a helper function for executing steps that can either be run
# locally or run under Docker. docker.sh, below, redefines it as "docker exec".
@@ -34,10 +38,16 @@ tfrun() { "$@"; }
# For Google-internal jobs, run copybara, which will overwrite the source tree.
# Never useful for outside users.
[[ "$TFCI_COPYBARA_ENABLE" == 1 ]] && source ./ci/official/utilities/copybara.sh
if [[ "$TFCI_COPYBARA_ENABLE" == 1 ]]; then
source ./ci/official/utilities/copybara.sh
fi
# Run all "tfrun" commands under Docker. See docker.sh for details
[[ "$TFCI_DOCKER_ENABLE" == 1 ]] && source ./ci/official/utilities/docker.sh
if [[ "$TFCI_DOCKER_ENABLE" == 1 ]]; then
source ./ci/official/utilities/docker.sh
fi
# Generate an overview page describing the build
[[ "$TFCI_INDEX_HTML_ENABLE" == 1 ]] && ./ci/official/utilities/generate_index_html.sh build/index.html
if [[ "$TFCI_INDEX_HTML_ENABLE" == 1 ]]; then
./ci/official/utilities/generate_index_html.sh build/index.html
fi
ci/official/wheel.sh
+8 -4
View File
@@ -2,12 +2,16 @@
source "${BASH_SOURCE%/*}/utilities/setup.sh"
# Record GPU count and CUDA version status
[[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]] && tfrun nvidia-smi
if [[ "$TFCI_NVIDIA_SMI_ENABLE" == 1 ]]; then
tfrun nvidia-smi
fi
# Update the version numbers for Nightly only
[[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]] && tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly
if [[ "$TFCI_NIGHTLY_UPDATE_VERSION_ENABLE" == 1 ]]; then
tfrun python3 tensorflow/tools/ci_build/update_version.py --nightly
fi
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_CACHE_ARGS[@]}" //tensorflow/tools/pip_package:build_pip_package
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" build "${TFCI_BAZEL_COMMON_ARGS[@]}" //tensorflow/tools/pip_package:build_pip_package
tfrun ./bazel-bin/tensorflow/tools/pip_package/build_pip_package build "${TFCI_BUILD_PIP_PACKAGE_ARGS[@]}"
tfrun ./ci/official/utilities/rename_and_verify_wheels.sh build
@@ -16,4 +20,4 @@ if [[ "$TFCI_UPLOAD_ENABLE" == 1 ]]; then
gsutil cp build/*.whl "$TFCI_UPLOAD_GCS_DESTINATION"
fi
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_CACHE_ARGS[@]}" --config=nonpip
tfrun bazel "${TFCI_BAZEL_BAZELRC_ARGS[@]}" test "${TFCI_BAZEL_COMMON_ARGS[@]}" --config=nonpip