From 0fae071b261867b712e1669eb09b3ec00a6810b4 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Thu, 10 Aug 2023 13:19:16 -0700 Subject: [PATCH] Add cleanup, resultstore options & fix bugs This bundles a few changes: - Added --config=resultstore option to all bazel invocations, which will create easy-to-read resultstore logs for individual invocations in official jobs - Fix an issue in rename_and_verify_wheels.sh where the wheel would be deleted - Copy all script output for all scripts to build/script.log for easier review and post-processing - Renamed DOCKER_GPU_ARGS to DOCKER_ARGS, since it's a convenient place to put other args as well, such as volume mounts - Added cleanup scripts that dump useful information after a job is complete, including a list of any remote resultstore invocations for easy reading - Re-use the same docker container if it still exists, which makes re-running easier PiperOrigin-RevId: 555625021 --- ci/official/bazelrcs/cpu.bazelrc | 13 +++++---- ci/official/bazelrcs/cuda.bazelrc | 12 +++++--- .../envs/continuous_linux_x86_cpu_py310 | 2 +- .../envs/continuous_linux_x86_cpu_py311 | 2 +- .../envs/continuous_linux_x86_cpu_py39 | 2 +- .../envs/continuous_linux_x86_cuda_py310 | 2 +- .../envs/continuous_linux_x86_cuda_py311 | 2 +- .../envs/continuous_linux_x86_cuda_py39 | 2 +- ci/official/envs/local_cpu | 2 +- .../envs/nightly_libtensorflow_linux_x86_cpu | 4 +-- .../envs/nightly_libtensorflow_linux_x86_cuda | 4 +-- ci/official/envs/nightly_linux_x86_cpu_py310 | 4 +-- ci/official/envs/nightly_linux_x86_cpu_py311 | 4 +-- ci/official/envs/nightly_linux_x86_cpu_py39 | 4 +-- ci/official/envs/nightly_linux_x86_cuda_py310 | 4 +-- ci/official/envs/nightly_linux_x86_cuda_py311 | 4 +-- ci/official/envs/nightly_linux_x86_cuda_py39 | 4 +-- ci/official/envs/nightly_linux_x86_tpu_py310 | 4 +-- ci/official/utilities/cleanup_docker.sh | 28 +++++++++++++++++++ ci/official/utilities/cleanup_summary.sh | 23 +++++++++++++++ ci/official/utilities/docker.sh | 14 +++++++--- .../utilities/rename_and_verify_wheels.sh | 5 ++-- ci/official/utilities/setup.sh | 15 ++++++++++ 23 files changed, 120 insertions(+), 40 deletions(-) create mode 100755 ci/official/utilities/cleanup_docker.sh create mode 100755 ci/official/utilities/cleanup_summary.sh diff --git a/ci/official/bazelrcs/cpu.bazelrc b/ci/official/bazelrcs/cpu.bazelrc index f5094ce2289..feef020e8dc 100644 --- a/ci/official/bazelrcs/cpu.bazelrc +++ b/ci/official/bazelrcs/cpu.bazelrc @@ -79,11 +79,15 @@ build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz # For outputting Build Event Protocol files build:build_event_export --build_event_json_file=build/bep.json +# Allow creation of resultstore URLs for any bazel invocation +build:resultstore --google_default_credentials +build:resultstore --bes_backend=buildeventservice.googleapis.com +build:resultstore --bes_results_url="https://source.cloud.google.com/results/invocations" +build:resultstore --bes_timeout=600s +build:resultstore --bes_instance_name="tensorflow-testing" + # For Remote Build Execution. -build:rbe --google_default_credentials -build:rbe --bes_backend=buildeventservice.googleapis.com -build:rbe --bes_results_url="https://source.cloud.google.com/results/invocations" -build:rbe --bes_timeout=600s +build:rbe --config=resultstore build:rbe --define=EXECUTOR=remote build:rbe --jobs=800 build:rbe --remote_executor=grpcs://remotebuildexecution.googleapis.com @@ -101,7 +105,6 @@ build:rbe --platforms="@sigbuild-r2.14-clang_config_platform//:platform" # Python config is the same across all containers because the binary is the same build:rbe --repo_env=TF_PYTHON_CONFIG_REPO="@sigbuild-r2.14-clang_config_python" build:rbe --remote_instance_name=projects/tensorflow-testing/instances/default_instance -build:rbe --project_id="tensorflow-testing" # For continuous builds test:pycpp_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only diff --git a/ci/official/bazelrcs/cuda.bazelrc b/ci/official/bazelrcs/cuda.bazelrc index f90cd2a5d38..1c0b5c55a3c 100644 --- a/ci/official/bazelrcs/cuda.bazelrc +++ b/ci/official/bazelrcs/cuda.bazelrc @@ -102,11 +102,15 @@ build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz # For outputting Build Event Protocol files build:build_event_export --build_event_json_file=build/bep.json +# Allow creation of resultstore URLs for any bazel invocation +build:resultstore --google_default_credentials +build:resultstore --bes_backend=buildeventservice.googleapis.com +build:resultstore --bes_results_url="https://source.cloud.google.com/results/invocations" +build:resultstore --bes_timeout=600s +build:resultstore --bes_instance_name="tensorflow-testing" + # For Remote Build Execution. -build:rbe --google_default_credentials -build:rbe --bes_backend=buildeventservice.googleapis.com -build:rbe --bes_results_url="https://source.cloud.google.com/results/invocations" -build:rbe --bes_timeout=600s +build:rbe --config=resultstore build:rbe --define=EXECUTOR=remote build:rbe --jobs=800 build:rbe --remote_executor=grpcs://remotebuildexecution.googleapis.com diff --git a/ci/official/envs/continuous_linux_x86_cpu_py310 b/ci/official/envs/continuous_linux_x86_cpu_py310 index 01c66e20988..1b5a10b6de1 100644 --- a/ci/official/envs/continuous_linux_x86_cpu_py310 +++ b/ci/official/envs/continuous_linux_x86_cpu_py310 @@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.10) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/continuous_linux_x86_cpu_py311 b/ci/official/envs/continuous_linux_x86_cpu_py311 index 0855cbe6c73..0ddc5cb9226 100644 --- a/ci/official/envs/continuous_linux_x86_cpu_py311 +++ b/ci/official/envs/continuous_linux_x86_cpu_py311 @@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.11) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/continuous_linux_x86_cpu_py39 b/ci/official/envs/continuous_linux_x86_cpu_py39 index c9836fc4ded..3b5adc17147 100644 --- a/ci/official/envs/continuous_linux_x86_cpu_py39 +++ b/ci/official/envs/continuous_linux_x86_cpu_py39 @@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.9) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/continuous_linux_x86_cuda_py310 b/ci/official/envs/continuous_linux_x86_cuda_py310 index e001597fcd3..aa888f44bf8 100644 --- a/ci/official/envs/continuous_linux_x86_cuda_py310 +++ b/ci/official/envs/continuous_linux_x86_cuda_py310 @@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.10) TFCI_BUILD_PIP_PACKAGE_ARGS=() TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/continuous_linux_x86_cuda_py311 b/ci/official/envs/continuous_linux_x86_cuda_py311 index 0da56e2ba02..60f785616be 100644 --- a/ci/official/envs/continuous_linux_x86_cuda_py311 +++ b/ci/official/envs/continuous_linux_x86_cuda_py311 @@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.11) TFCI_BUILD_PIP_PACKAGE_ARGS=() TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/continuous_linux_x86_cuda_py39 b/ci/official/envs/continuous_linux_x86_cuda_py39 index 9ea2c57867e..44f4180f17f 100644 --- a/ci/official/envs/continuous_linux_x86_cuda_py39 +++ b/ci/official/envs/continuous_linux_x86_cuda_py39 @@ -5,7 +5,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config rbe --repo_env=TF_PYTHON_VERSION=3.9) TFCI_BUILD_PIP_PACKAGE_ARGS=() TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/local_cpu b/ci/official/envs/local_cpu index 6acb9e80f1a..3f383563515 100644 --- a/ci/official/envs/local_cpu +++ b/ci/official/envs/local_cpu @@ -3,7 +3,7 @@ TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache --repo_env=TF_PYTHON_VERS TFCI_BUILD_PIP_PACKAGE_ARGS=("--cpu") TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9 TFCI_DOCKER_PULL_ENABLE= TFCI_GIT_DIR=. diff --git a/ci/official/envs/nightly_libtensorflow_linux_x86_cpu b/ci/official/envs/nightly_libtensorflow_linux_x86_cpu index 8d8747e0365..ac4f3e02c87 100644 --- a/ci/official/envs/nightly_libtensorflow_linux_x86_cpu +++ b/ci/official/envs/nightly_libtensorflow_linux_x86_cpu @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_libtensorflow_linux_x86_cuda b/ci/official/envs/nightly_libtensorflow_linux_x86_cuda index 048acf38645..9a6a8616166 100644 --- a/ci/official/envs/nightly_libtensorflow_linux_x86_cuda +++ b/ci/official/envs/nightly_libtensorflow_linux_x86_cuda @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10) TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_cpu_py310 b/ci/official/envs/nightly_linux_x86_cpu_py310 index 3ad0420b466..8a180a50c4f 100644 --- a/ci/official/envs/nightly_linux_x86_cpu_py310 +++ b/ci/official/envs/nightly_linux_x86_cpu_py310 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_cpu_py311 b/ci/official/envs/nightly_linux_x86_cpu_py311 index f0455007974..b92e4de1857 100644 --- a/ci/official/envs/nightly_linux_x86_cpu_py311 +++ b/ci/official/envs/nightly_linux_x86_cpu_py311 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.11) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.11) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_cpu_py39 b/ci/official/envs/nightly_linux_x86_cpu_py39 index 5d7a2e657e7..9dfbff45ce9 100644 --- a/ci/official/envs/nightly_linux_x86_cpu_py39 +++ b/ci/official/envs/nightly_linux_x86_cpu_py39 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.9) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.9) TFCI_BUILD_PIP_PACKAGE_ARGS=(--cpu --nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_cuda_py310 b/ci/official/envs/nightly_linux_x86_cuda_py310 index 59c672e9f31..27fb2548ab3 100644 --- a/ci/official/envs/nightly_linux_x86_cuda_py310 +++ b/ci/official/envs/nightly_linux_x86_cuda_py310 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10) TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_cuda_py311 b/ci/official/envs/nightly_linux_x86_cuda_py311 index 87a1f75cdd1..26c7a6f7cae 100644 --- a/ci/official/envs/nightly_linux_x86_cuda_py311 +++ b/ci/official/envs/nightly_linux_x86_cuda_py311 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.11) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.11) TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.11 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_cuda_py39 b/ci/official/envs/nightly_linux_x86_cuda_py39 index 7c4622c24a2..353fb0ca5db 100644 --- a/ci/official/envs/nightly_linux_x86_cuda_py39 +++ b/ci/official/envs/nightly_linux_x86_cuda_py39 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cuda.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.9) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.9) TFCI_BUILD_PIP_PACKAGE_ARGS=(--nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=(--gpus all) +TFCI_DOCKER_ARGS=(--gpus all) TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.9 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/envs/nightly_linux_x86_tpu_py310 b/ci/official/envs/nightly_linux_x86_tpu_py310 index d7b362985e3..fabc2a80def 100644 --- a/ci/official/envs/nightly_linux_x86_tpu_py310 +++ b/ci/official/envs/nightly_linux_x86_tpu_py310 @@ -1,11 +1,11 @@ #TFCI_UPLOAD_LIB_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" #TFCI_UPLOAD_WHL_GCS_URI="gs://tensorflow-release-packages/$RELEASE_VERSION/$KOKORO_GIT_COMMIT_tensorflow" TFCI_BAZEL_BAZELRC_ARGS=(--bazelrc ./ci/official/bazelrcs/cpu.bazelrc) -TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --repo_env=TF_PYTHON_VERSION=3.10 --define=with_tpu_support=true) +TFCI_BAZEL_COMMON_ARGS=(--config sigbuild_remote_cache_push --config resultstore --repo_env=TF_PYTHON_VERSION=3.10 --define=with_tpu_support=true) TFCI_BUILD_PIP_PACKAGE_ARGS=(--tpu --nightly_flag) TFCI_COPYBARA_ENABLE=0 TFCI_DOCKER_ENABLE=1 -TFCI_DOCKER_GPU_ARGS=() +TFCI_DOCKER_ARGS=() TFCI_DOCKER_IMAGE=tensorflow/build:latest-python3.10 TFCI_DOCKER_PULL_ENABLE=1 TFCI_GIT_DIR=$KOKORO_ARTIFACTS_DIR/github/tensorflow diff --git a/ci/official/utilities/cleanup_docker.sh b/ci/official/utilities/cleanup_docker.sh new file mode 100755 index 00000000000..b030249b3f0 --- /dev/null +++ b/ci/official/utilities/cleanup_docker.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2023 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +cat </dev/null 2>&1 ; then + docker run "${TFCI_DOCKER_ARGS[@]}" --name tf -w "$TFCI_GIT_DIR" -itd --rm \ + -v "$TFCI_GIT_DIR:$TFCI_GIT_DIR" \ + "$TFCI_DOCKER_IMAGE" \ + bash +fi tfrun() { docker exec tf "$@"; } diff --git a/ci/official/utilities/rename_and_verify_wheels.sh b/ci/official/utilities/rename_and_verify_wheels.sh index 72b26ea56d1..41c69792120 100755 --- a/ci/official/utilities/rename_and_verify_wheels.sh +++ b/ci/official/utilities/rename_and_verify_wheels.sh @@ -20,12 +20,13 @@ set -euxo pipefail DIR=$1 -for wheel in $DIR/*.whl; do +find $DIR -iname "*.whl" | while read wheel; do echo "Checking and renaming $wheel..." + wheel=$(realpath "$wheel") time python3 -m auditwheel repair --plat manylinux2014_x86_64 "$wheel" --wheel-dir build 2>&1 | tee check.txt # We don't need the original wheel if it was renamed - new_wheel=$(grep --extended-regexp --only-matching '\S+.whl' check.txt | tail -n 1) + new_wheel=$(awk '/Fixed-up wheel written to/ {print $NF}' check.txt) if [[ "$new_wheel" != "$wheel" ]]; then rm "$wheel" wheel="$new_wheel" diff --git a/ci/official/utilities/setup.sh b/ci/official/utilities/setup.sh index 2f91ec4d010..fc793a8b69a 100755 --- a/ci/official/utilities/setup.sh +++ b/ci/official/utilities/setup.sh @@ -50,6 +50,11 @@ fi cd "$TFCI_GIT_DIR" mkdir -p build +# In addition to dumping all script output to the terminal, place it into +# build/script.log +rm build/script.log +exec > >(tee "build/script.log") 2>&1 + # Setup tfrun, a helper function for executing steps that can either be run # locally or run under Docker. docker.sh, below, redefines it as "docker exec". # Important: "tfrun foo | bar" is "( tfrun foo ) | bar", not tfrun (foo | bar). @@ -85,3 +90,13 @@ fi if [[ "$TFCI_INDEX_HTML_ENABLE" == 1 ]]; then ./ci/official/utilities/generate_index_html.sh build/index.html fi + +# Single handler for all cleanup actions, triggered on an EXIT trap. +# TODO(angerson) Making this use different scripts may be overkill. +cleanup() { + if [[ "$TFCI_DOCKER_ENABLE" == 1 ]]; then + ./ci/official/utilities/cleanup_docker.sh + fi + ./ci/official/utilities/cleanup_summary.sh +} +trap cleanup EXIT