From 7f146afcd3a825c9db6963b372974b3e15ec5b52 Mon Sep 17 00:00:00 2001 From: Alexey Sokolov Date: Thu, 25 May 2017 00:02:09 +0100 Subject: [PATCH] Use Travis stages Fix #1407 --- .travis.yml | 43 +++++++------ .travis_after_all.py | 145 ------------------------------------------- 2 files changed, 23 insertions(+), 165 deletions(-) delete mode 100644 .travis_after_all.py diff --git a/.travis.yml b/.travis.yml index 4ba4beb7..0fbae429 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,8 +54,30 @@ matrix: - os: linux compiler: gcc env: BUILD_TYPE=tarball BUILD_WITH=autoconf COVERAGE=gcov + - stage: deploy + os: linux + before_install: + install: + - if [[ "$TRAVIS_REPO_SLUG" == "znc/znc" && "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_BRANCH" == "master" ]]; then ATTEMPT_DEPLOY=yes; else ATTEMPT_DEPLOY=no; fi + - if [[ "$ATTEMPT_DEPLOY" == "yes" ]]; then openssl aes-256-cbc -d -in .travis-github.enc -out ~/znc-github-key -k ${SECRET_KEY}; fi + - export SECRET_KEY=no + - if [[ "$ATTEMPT_DEPLOY" == "yes" ]]; then sudo apt-get update; fi + - if [[ "$ATTEMPT_DEPLOY" == "yes" ]]; then sudo apt-get install -y doxygen graphviz; fi + script: + - | + if [[ "$ATTEMPT_DEPLOY" == "yes" ]]; then + mkdir -p ~/.ssh + chmod 0600 ~/znc-github-key + cp .travis.ssh ~/.ssh/config + git config --global user.email "travis-ci@znc.in" + git config --global user.name "znc-travis" + fi + - if [[ "$ATTEMPT_DEPLOY" == "yes" ]]; then ./.travis-generate-docs.sh; fi + - if [[ "$ATTEMPT_DEPLOY" == "yes" ]]; then ./.travis-coverity-scan.py; fi + after_success: before_install: - "echo os: [$TRAVIS_OS_NAME] build: [$BUILD_TYPE]" + - export SECRET_KEY=no - export CFGFLAGS= MYCXXFLAGS= MYLDFLAGS= - if [[ "$BUILD_TYPE" == "tarball" ]]; then CFGFLAGS+=" --with-gtest=/usr/src/gtest --with-gmock=/usr/src/gmock --disable-swig"; fi - if [[ "$BUILD_TYPE" == "asan" ]]; then MYCXXFLAGS+=" -fsanitize=address -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fPIE" MYLDFLAGS+=" -fsanitize=address -pie"; fi @@ -65,8 +87,6 @@ before_install: - if [[ "$COVERAGE" == "llvm" ]]; then MYCXXFLAGS+=" -fprofile-instr-generate -fcoverage-mapping" MYLDFLAGS+=" -fprofile-instr-generate"; fi # UBSan randomly crashes clang, and very often :( # CFGFLAGS= MYCXXFLAGS="-fsanitize=undefined -O1 -fPIE -fno-sanitize-recover" MYLDFLAGS="-fsanitize=undefined -pie -fno-sanitize-recover" - - if [[ "$TRAVIS_REPO_SLUG" == "znc/znc" && "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_BRANCH" == "master" ]]; then openssl aes-256-cbc -d -in .travis-github.enc -out ~/znc-github-key -k ${SECRET_KEY}; fi - - export SECRET_KEY=no install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then cat /proc/cpuinfo /proc/meminfo; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then lsb_release -a; fi @@ -75,7 +95,7 @@ install: - if [[ "$TRAVIS_OS_NAME" == "linux" && "$BUILD_TYPE" == "tarball" ]]; then sudo add-apt-repository -y ppa:ostogvin/tjo-develop; fi # gtest+gmock - if [[ "$TRAVIS_OS_NAME" == "linux" && "$BUILD_WITH" == "cmake" ]]; then sudo add-apt-repository -y ppa:george-edison55/cmake-3.x; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get update; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y libperl-dev python3-dev tcl-dev libsasl2-dev libicu-dev swig3.0 doxygen graphviz qt55base python3-yaml libboost-locale-dev; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y libperl-dev python3-dev tcl-dev libsasl2-dev libicu-dev swig3.0 qt55base python3-yaml libboost-locale-dev; fi - if [[ "$TRAVIS_OS_NAME" == "linux" && "$BUILD_TYPE" == "tarball" ]]; then sudo apt-get install -y google-mock libgtest-dev; fi # Clang 3.5 TSan is broken on Travis Ubuntu 14.04. Clang 3.8 seems to work, but only without -pie (https://github.com/google/sanitizers/issues/503) - if [[ "$TRAVIS_OS_NAME" == "linux" && "$BUILD_TYPE" == "tsan" ]]; then sudo apt-get install -y clang-3.8; export CC=clang-3.8 CXX=clang++-3.8; fi @@ -117,23 +137,6 @@ after_success: find /usr/local/lib/znc -name '*.so' -or -name '*.bundle' | while read f; do xcrun llvm-cov show -show-line-counts-or-regions -instr-profile=inttest.profdata $f > inttest-$(basename $f)-coverage.txt; done fi - bash <(curl -s https://codecov.io/bash) - - cd "$TRAVIS_BUILD_DIR" - - test -r "$TRAVIS_BUILD_DIR/.travis_after_all.py" && python "$TRAVIS_BUILD_DIR/.travis_after_all.py" || echo No .travis_after_all.py found - - test -r .to_export_back && echo $(cat .to_export_back) || echo No .to_export_back found - - export DUMMY_VAR=dummy-value $(cat .to_export_back) - - | - if [ "$BUILD_LEADER" == "YES" ] && [ "$BUILD_AGGREGATE_STATUS" == "others_succeeded" ] && [ "$TRAVIS_REPO_SLUG" == "znc/znc" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_BRANCH" == "master" ]; then - echo "All builds passed!" - mkdir -p ~/.ssh - chmod 0600 ~/znc-github-key - cp .travis.ssh ~/.ssh/config - git config --global user.email "travis-ci@znc.in" - git config --global user.name "znc-travis" - ./.travis-generate-docs.sh - ./.travis-coverity-scan.py - else - echo "Not leader" - fi notifications: irc: channels: diff --git a/.travis_after_all.py b/.travis_after_all.py deleted file mode 100644 index 3b46dc77..00000000 --- a/.travis_after_all.py +++ /dev/null @@ -1,145 +0,0 @@ -# https://github.com/dmakhno/travis_after_all with MIT license - -import os -import sys -import json -import time -import logging - -try: - from functools import reduce -except ImportError: - pass - -try: - import urllib.request as urllib2 -except ImportError: - import urllib2 - -log = logging.getLogger("travis.leader") -log.addHandler(logging.StreamHandler()) -log.setLevel(logging.INFO) - -TRAVIS_JOB_NUMBER = 'TRAVIS_JOB_NUMBER' -TRAVIS_BUILD_ID = 'TRAVIS_BUILD_ID' -POLLING_INTERVAL = 'LEADER_POLLING_INTERVAL' -GITHUB_TOKEN = 'GITHUB_TOKEN' - - -# Travis API entry point, there are at least https://api.travis-ci.com and https://api.travis-ci.org -travis_entry = sys.argv[1] if len(sys.argv) > 1 else 'https://api.travis-ci.org' - -build_id = os.getenv(TRAVIS_BUILD_ID) -polling_interval = int(os.getenv(POLLING_INTERVAL, '5')) -gh_token = os.getenv(GITHUB_TOKEN) - - -my_job_number = os.getenv(TRAVIS_JOB_NUMBER) -if not my_job_number: - # seems even for builds with only one job, this won't get here - log.fatal("Don't use defining leader for build without matrix") - exit(1) - -leader_name = '' - - -def is_leader(job): - return job == leader_name - - -def detect_leader(token): - global leader_name - matrix = matrix_snapshot(token) - leader_name = matrix[-1].number - - -class MatrixElement(object): - - def __init__(self, json_raw): - self.is_finished = json_raw['finished_at'] is not None - self.is_succeeded = json_raw['result'] == 0 - self.number = json_raw['number'] - self.is_leader = is_leader(self.number) - - -def matrix_snapshot(travis_token): - """ - :return: Matrix List - """ - headers = {'content-type': 'application/json'} - if travis_token is not None: - headers['Authorization'] = 'token {}'.format(travis_token) - req = urllib2.Request("{0}/builds/{1}".format(travis_entry, build_id), headers=headers) - response = urllib2.urlopen(req).read() - raw_json = json.loads(response.decode('utf-8')) - matrix_without_leader = [MatrixElement(job) for job in raw_json["matrix"] if not is_leader(job['number'])] - return matrix_without_leader - - -def wait_others_to_finish(travis_token): - def others_finished(): - """ - Dumps others to finish - Leader cannot finish, it is working now - :return: tuple(True or False, List of not finished jobs) - """ - snapshot = matrix_snapshot(travis_token) - finished = [job.is_finished for job in snapshot if not job.is_leader] - return reduce(lambda a, b: a and b, finished), [job.number for job in snapshot if - not job.is_leader and not job.is_finished] - - while True: - finished, waiting_list = others_finished() - if finished: - break - log.info("Leader waits for minions {0}...".format(waiting_list)) # just in case do not get "silence timeout" - time.sleep(polling_interval) - - -def get_token(): - if gh_token is None or gh_token == '': - return None - data = {"github_token": gh_token} - headers = {'content-type': 'application/json', 'User-Agent': 'Travis/1.0'} - - req = urllib2.Request("{0}/auth/github".format(travis_entry), json.dumps(data).encode('utf-8'), headers) - response = urllib2.urlopen(req).read() - travis_token = json.loads(response.decode('utf-8')).get('access_token') - - return travis_token - - -try: - token = get_token() - - detect_leader(token) - if is_leader(my_job_number): - log.info("This is a leader") - else: - # since python is subprocess, env variables are exported back via file - with open(".to_export_back", "w") as export_var: - export_var.write("BUILD_MINION=YES") - log.info("This is a minion") - exit(0) - - wait_others_to_finish(token) - - final_snapshot = matrix_snapshot(token) - log.info("Final Results: {0}".format([(e.number, e.is_succeeded) for e in final_snapshot])) - - BUILD_AGGREGATE_STATUS = 'BUILD_AGGREGATE_STATUS' - others_snapshot = [el for el in final_snapshot if not el.is_leader] - if reduce(lambda a, b: a and b, [e.is_succeeded for e in others_snapshot]): - os.environ[BUILD_AGGREGATE_STATUS] = "others_succeeded" - elif reduce(lambda a, b: a and b, [not e.is_succeeded for e in others_snapshot]): - log.error("Others Failed") - os.environ[BUILD_AGGREGATE_STATUS] = "others_failed" - else: - log.warn("Others Unknown") - os.environ[BUILD_AGGREGATE_STATUS] = "unknown" - # since python is subprocess, env variables are exported back via file - with open(".to_export_back", "w") as export_var: - export_var.write("BUILD_LEADER=YES {0}={1}".format(BUILD_AGGREGATE_STATUS, os.environ[BUILD_AGGREGATE_STATUS])) - -except Exception as e: - log.fatal(e)