zzos/tensorflow
1
0
Fork 0
mirror of https://github.com/tensorflow/tensorflow.git synced 2026-09-28 13:23:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add cleanup, resultstore options & fix bugs

Browse Source 
This bundles a few changes:

- Added --config=resultstore option to all bazel invocations, which will
  create easy-to-read resultstore logs for individual invocations in official
  jobs
- Fix an issue in rename_and_verify_wheels.sh where the wheel would be deleted
- Copy all script output for all scripts to build/script.log for easier review
  and post-processing
- Renamed DOCKER_GPU_ARGS to DOCKER_ARGS, since it's a convenient place to put
  other args as well, such as volume mounts
- Added cleanup scripts that dump useful information after a job is complete,
  including a list of any remote resultstore invocations for easy reading
- Re-use the same docker container if it still exists, which makes re-running
  easier

PiperOrigin-RevId: 555625021
This commit is contained in:
Austin Anderson
2023-08-10 13:26:29 -07:00
committed by TensorFlower Gardener
parent 204b924e1a
commit 0fae071b26
23 changed files with 120 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/official/bazelrcs/cpu.bazelrc
+8 -5
View File
@@ -79,11 +79,15 @@ build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=build/bep.json
# Allow creation of resultstore URLs for any bazel invocation
build:resultstore --google_default_credentials
build:resultstore --bes_backend=buildeventservice.googleapis.com
build:resultstore --bes_results_url="https://source.cloud.google.com/results/invocations"
build:resultstore --bes_timeout=600s
build:resultstore --bes_instance_name="tensorflow-testing"
# For Remote Build Execution.
build:rbe --google_default_credentials
build:rbe --bes_backend=buildeventservice.googleapis.com
build:rbe --bes_results_url="https://source.cloud.google.com/results/invocations"
build:rbe --bes_timeout=600s
build:rbe --config=resultstore
build:rbe --define=EXECUTOR=remote
build:rbe --jobs=800
build:rbe --remote_executor=grpcs://remotebuildexecution.googleapis.com
@@ -101,7 +105,6 @@ build:rbe --platforms="@sigbuild-r2.14-clang_config_platform//:platform"
# Python config is the same across all containers because the binary is the same
build:rbe --repo_env=TF_PYTHON_CONFIG_REPO="@sigbuild-r2.14-clang_config_python"
build:rbe --remote_instance_name=projects/tensorflow-testing/instances/default_instance
build:rbe --project_id="tensorflow-testing"
# For continuous builds
test:pycpp_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
ci/official/bazelrcs/cuda.bazelrc
+8 -4
View File
@@ -102,11 +102,15 @@ build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=build/bep.json
# Allow creation of resultstore URLs for any bazel invocation
build:resultstore --google_default_credentials
build:resultstore --bes_backend=buildeventservice.googleapis.com
build:resultstore --bes_results_url="https://source.cloud.google.com/results/invocations"
build:resultstore --bes_timeout=600s
build:resultstore --bes_instance_name="tensorflow-testing"
# For Remote Build Execution.
build:rbe --google_default_credentials
build:rbe --bes_backend=buildeventservice.googleapis.com
build:rbe --bes_results_url="https://source.cloud.google.com/results/invocations"
build:rbe --bes_timeout=600s
build:rbe --config=resultstore
build:rbe --define=EXECUTOR=remote
build:rbe --jobs=800
build:rbe --remote_executor=grpcs://remotebuildexecution.googleapis.com
ci/official/envs/continuous_linux_x86_cpu_py310
+1 -1
View File
@@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/continuous_linux_x86_cpu_py311
+1 -1
View File
@@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.11)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/continuous_linux_x86_cpu_py39
+1 -1
View File
@@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.9)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/continuous_linux_x86_cuda_py310
+1 -1
View File
@@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BUILD_PIP_PACKAGE_ARGS=()
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/continuous_linux_x86_cuda_py311
+1 -1
View File
@@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.11)
TFCI_BUILD_PIP_PACKAGE_ARGS=()
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/continuous_linux_x86_cuda_py39
+1 -1
View File
@@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.9)
TFCI_BUILD_PIP_PACKAGE_ARGS=()
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/local_cpu
+1 -1
View File
@@ -3,7 +3,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache --repo_env=TF_PYTHON_VERS
TFCI_BUILD_PIP_PACKAGE_ARGS=("--cpu")
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9
TFCI_DOCKER_PULL_ENABLE=
TFCI_GIT_DIR=.
ci/official/envs/nightly_libtensorflow_linux_x86_cpu
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_libtensorflow_linux_x86_cuda
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_cpu_py310
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_cpu_py311
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.11)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.11)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_cpu_py39
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.9)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.9)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_cuda_py310
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_cuda_py311
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.11)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.11)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_cuda_py39
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.9)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.9)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=(--gpus all)
TFCI_DOCKER_ARGS=(--gpus all)
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/envs/nightly_linux_x86_tpu_py310
+2 -2
View File
@@ -1,11 +1,11 @@
#TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
#TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow"
TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10 --define=with_tpu_support=true)
TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10 --define=with_tpu_support=true)
TFCI_BUILD_PIP_PACKAGE_ARGS=(--tpu --nightly_flag)
TFCI_COPYBARA_ENABLE=0
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_GPU_ARGS=()
TFCI_DOCKER_ARGS=()
TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10
TFCI_DOCKER_PULL_ENABLE=1
TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow
ci/official/utilities/cleanup_docker.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
cat <<EOF
IMPORTANT: These tests ran under docker. This script does not clean up the
container for you! You can delete the container with:
$ docker rm -f tf
You can also execute more commands within the container with e.g.:
$ docker exec tf bazel clean
$ docker exec -it tf bash
EOF
docker ps
ci/official/utilities/cleanup_summary.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
cat <<EOF
IMPORTANT: For bazel invocations that uploaded to resultstore (e.g. RBE), you
can view more detailed results that are probably easier to read than this log.
Try the links below:
EOF
# Find any "Streaming build results to" line, then print the last word in it
awk '/Streaming build results to/ {print $NF}' $TFCI_GIT_DIR/build/script.log | uniq
ci/official/utilities/docker.sh
+10 -4
View File
@@ -16,8 +16,14 @@
if [[ "$TFCI_DOCKER_PULL_ENABLE" == 1 ]]; then
docker pull "$TFCI_DOCKER_IMAGE"
fi
docker run "${TFCI_DOCKER_GPU_ARGS[@]}" --name tf -w "$TFCI_GIT_DIR" -itd --rm \
-v "$TFCI_GIT_DIR:$TFCI_GIT_DIR" \
"$TFCI_DOCKER_IMAGE" \
bash
# Keep the existing "tf" container if it's already present.
# The container is not cleaned up automatically! Remove it with:
# docker rm tf
if ! docker container inspect tf >/dev/null 2>&1 ; then
docker run "${TFCI_DOCKER_ARGS[@]}" --name tf -w "$TFCI_GIT_DIR" -itd --rm \
-v "$TFCI_GIT_DIR:$TFCI_GIT_DIR" \
"$TFCI_DOCKER_IMAGE" \
bash
fi
tfrun() { docker exec tf "$@"; }
ci/official/utilities/rename_and_verify_wheels.sh
+3 -2
View File
@@ -20,12 +20,13 @@
set -euxo pipefail
DIR=$1
for wheel in $DIR/*.whl; do
find $DIR -iname "*.whl" | while read wheel; do
echo "Checking and renaming $wheel..."
wheel=$(realpath "$wheel")
time python3 -m auditwheel repair --plat manylinux2014_x86_64 "$wheel" --wheel-dir build 2>&1 | tee check.txt
# We don't need the original wheel if it was renamed
new_wheel=$(grep --extended-regexp --only-matching '\S+.whl' check.txt | tail -n 1)
new_wheel=$(awk '/Fixed-up wheel written to/ {print $NF}' check.txt)
if [[ "$new_wheel" != "$wheel" ]]; then
rm "$wheel"
wheel="$new_wheel"
ci/official/utilities/setup.sh
+15
View File
@@ -50,6 +50,11 @@ fi
cd "$TFCI_GIT_DIR"
mkdir -p build
# In addition to dumping all script output to the terminal, place it into
# build/script.log
rm build/script.log
exec > >(tee "build/script.log") 2>&1
# Setup tfrun, a helper function for executing steps that can either be run
# locally or run under Docker. docker.sh, below, redefines it as "docker exec".
# Important: "tfrun foo | bar" is "( tfrun foo ) | bar", not tfrun (foo | bar).
@@ -85,3 +90,13 @@ fi
if [[ "$TFCI_INDEX_HTML_ENABLE" == 1 ]]; then
./ci/official/utilities/generate_index_html.sh build/index.html
fi
# Single handler for all cleanup actions, triggered on an EXIT trap.
# TODO(angerson) Making this use different scripts may be overkill.
cleanup() {
if [[ "$TFCI_DOCKER_ENABLE" == 1 ]]; then
./ci/official/utilities/cleanup_docker.sh
fi
./ci/official/utilities/cleanup_summary.sh
}
trap cleanup EXIT