run_cperf

#!/usr/bin/env python
# ===--- run_cperf --------------------------------------------------------===
#
#  This source file is part of the Swift.org open source project
#
#  Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
#  Licensed under Apache License v2.0 with Runtime Library Exception
#
#  See https://swift.org/LICENSE.txt for license information
#  See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===----------------------------------------------------------------------===

"""A run script to be executed as a pull-request test on Jenkins."""

import argparse
import os
import platform
import re
import shutil
import sys

import common


OLD_INSTANCE = 'old'
NEW_INSTANCE = 'new'


def get_workspace_for_instance(instance, args):
    return common.private_workspace(
        os.path.join(args.swift_branch, instance))


def setup_workspace(instance, workspace, args):
    if args.clean:
        shutil.rmtree(workspace)
    if not os.path.exists(workspace):
        os.makedirs(workspace)
    swift = os.path.join(workspace, "swift")
    if not os.path.exists(swift):
        common.git_clone(args.url, swift, tree=args.swift_branch)
    common.check_execute(['utils/update-checkout',
                          '--clone', '--reset-to-remote',
                          '--clean', '--scheme', args.swift_branch],
                         cwd=swift)
    if instance == NEW_INSTANCE:
        command_fetch = ['git', '-C', swift, 'fetch', 'origin',
                         'pull/%d/merge' % args.setup_workspaces_for_pr]
        common.check_execute(command_fetch)
        common.git_checkout('FETCH_HEAD', swift)


def validate_workspace(instance, workspace, args):
    if not os.path.exists(workspace):
        raise RuntimeError("Missing %s workspace dir %s"
                           % (instance, workspace))
    swift = os.path.join(workspace, "swift")
    if not os.path.exists(swift):
        raise RuntimeError("Missing swift checkout in workspace dir %s"
                           % workspace)


def main():
    common.debug_print('** RUN PULL-REQUEST CPERF **')
    os.chdir(os.path.dirname(__file__))

    args = parse_args()
    instances = [NEW_INSTANCE, OLD_INSTANCE]
    configs = ['debug', 'debug-opt', 'wmo-onone', 'release']

    for instance in instances:
        workspace = get_workspace_for_instance(instance, args)

        if args.setup_workspaces_for_pr:
            setup_workspace(instance, workspace, args)
        validate_workspace(instance, workspace, args)

        if not args.skip_build:
            build_swift_toolchain(workspace, args)

        if not args.skip_runner:
            execute_runner(instance, workspace, configs, args)

    regressions = analyze_results(configs, args)

    # Temporary hack to write output to workspace when in CI,
    # regardless of --output passed.
    if 'WORKSPACE' in os.environ:
        workspace = os.environ['WORKSPACE']
        print("WORKSPACE: %s" % workspace)
        p = os.path.join(workspace, 'comment.md')
        o = os.path.abspath(os.path.join(os.getcwd(),
                                         args.output.name))
        print("Output written to: %s" % o)
        if o != p:
            print("Copying %s to %s" % (o, p))
            args.output.close()
            shutil.copyfile(o, p)

    return regressions


def get_sandbox_profile_flags():
    sandbox_flags = []
    sscss = "../../../workspace-private/swift-source-compat-suite-sandbox"
    if not os.path.exists(sscss):
        raise RuntimeError("Missing sandbox dir %s" % sscss)

    if platform.system() == 'Darwin':
        sandbox_flags += [
            '--sandbox-profile-xcodebuild',
            os.path.join(sscss, 'sandbox_xcodebuild.sb'),
            '--sandbox-profile-package',
            os.path.join(sscss, 'sandbox_package.sb')
        ]
    elif platform.system() == 'Linux':
        sandbox_flags += [
            '--sandbox-profile-package',
            os.path.join(sscss, 'sandbox_package_linux.profile')
        ]
    else:
        raise common.UnsupportedPlatform
    return sandbox_flags


def get_swiftc_path(workspace):
    if platform.system() == 'Darwin':
        swiftc_path = os.path.join(workspace, 'build/compat_macos/install/toolchain/usr/bin/swiftc')
    elif platform.system() == 'Linux':
        swiftc_path = os.path.join(workspace, 'build/compat_linux/install/usr/bin/swiftc')
    else:
        raise common.UnsupportedPlatform
    return swiftc_path


def build_swift_toolchain(workspace, args):
    if platform.system() == 'Darwin':
        build_command = [
            os.path.join(workspace, 'swift/utils/build-script'),
            '--release',
            '--no-assertions',
            '--build-ninja',
            '--llbuild',
            '--swiftpm',
            '--ios',
            '--tvos',
            '--watchos',
            '--skip-build-benchmarks',
            '--build-subdir=compat_macos',
            '--compiler-vendor=apple',
            '--',
            '--darwin-install-extract-symbols',
            '--darwin-toolchain-alias=swift',
            '--darwin-toolchain-bundle-identifier=org.swift.compat-macos',
            '--darwin-toolchain-display-name-short=Swift Development Snapshot'
            '--darwin-toolchain-display-name=Swift Development Snapshot',
            '--darwin-toolchain-name=swift-DEVELOPMENT-SNAPSHOT',
            '--darwin-toolchain-version=3.999.999',
            '--install-llbuild',
            '--install-swift',
            '--install-swiftpm',
            '--install-destdir={}/build/compat_macos/install'.format(workspace),
            '--install-prefix=/toolchain/usr',
            '--install-symroot={}/build/compat_macos/symroot'.format(workspace),
            '--installable-package={}/build/compat_macos/root.tar.gz'.format(workspace),
            '--llvm-install-components=libclang;libclang-headers',
            '--swift-install-components=compiler;clang-builtin-headers;stdlib;sdk-overlay;license;sourcekit-xpc-service;swift-remote-mirror;swift-remote-mirror-headers',
            '--symbols-package={}/build/compat_macos/root-symbols.tar.gz'.format(workspace),
            '--verbose-build',
            '--reconfigure',
        ]
    elif platform.system() == 'Linux':
        build_command = [
            os.path.join(workspace, 'swift/utils/build-script'),
            '--release',
            '--no-assertions',
            '--build-ninja',
            '--llbuild',
            '--swiftpm',
            '--foundation',
            '--libdispatch',
            '--xctest',
            '--skip-build-benchmarks',
            '--build-subdir=compat_linux',
            '--',
            '--install-foundation',
            '--install-libdispatch',
            '--install-llbuild',
            '--install-swift',
            '--install-swiftpm',
            '--install-xctest',
            '--install-destdir={}/build/compat_linux/install'.format(workspace),
            '--install-prefix=/usr',
            '--installable-package={}/build/compat_linux/root.tar.gz'.format(workspace),
            '--swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;swift-remote-mirror;sdk-overlay;license',
            '--verbose-build',
            '--reconfigure',
        ]
    else:
        raise common.UnsupportedPlatform
    common.check_execute(build_command, timeout=9999999)


def get_projects(suite):
    if suite == 'full':
        return 'projects.json'
    elif suite == 'smoketest':
        return 'projects-cperf-smoketest.json'
    else:
        raise ValueError("Unknown suite: " + suite)


def get_stats_dir(instance, variant):
    return os.path.join(os.getcwd(),
                        "-".join(["stats", instance, variant]))


def get_actual_config_and_flags(config, stats):
    flags = ("-stats-output-dir '%s'" % stats)
    # Handle as a pseudo-configs
    if config == 'wmo-onone':
        flags += ' -wmo -Onone '
        config = 'release'
    elif config == 'debug-opt':
        flags += ' -O '
        config = 'debug'
    return (config, flags)


def get_variant(config, args):
    return '-'.join([args.suite, args.swift_branch, config])


def execute_runner(instance, workspace, configs, args):
    projects = get_projects(args.suite)
    swiftc_path = get_swiftc_path(workspace)
    for config in configs:
        variant = get_variant(config, args)
        stats = get_stats_dir(instance, variant)
        if os.path.exists(stats):
            shutil.rmtree(stats)
        os.makedirs(stats)
        (config, flags) = get_actual_config_and_flags(config, stats)
        runner_command = [
            './runner.py',
            '--swiftc', swiftc_path,
            '--projects', projects,
            '--build-config', config,
            '--swift-version', '3',
            '--include-actions', 'action.startswith("Build")',
            '--swift-branch', args.swift_branch,
            '--add-swift-flags', flags,
        ]
        if args.sandbox:
            runner_command += get_sandbox_profile_flags()
        if args.verbose:
            runner_command += ["--verbose"]
        common.check_execute(runner_command, timeout=9999999)


def get_table_name(reference, subset, variant):
    return os.path.join(
        os.getcwd(),
        ('-'.join([reference, subset, variant]) + '.md'))


def get_config_desc(config):
    return config.capitalize()


def get_reference_desc(reference, config):
    return ("PR vs. %s (%s)" %
            (reference, config))


def get_table_desc(reference, subset, config):
    return ("PR vs. %s, changed %s (%s)" %
            (reference, subset, config))


def make_internal_link(desc):
    link = re.sub('[^ a-zA-Z0-9-]', '', desc)
    link = link.strip().lower()
    link = re.sub(' +', '-', link)
    return ("[%s](#%s)" % (desc, link))


def get_baseline_name(variant):
    return os.path.join(os.getcwd(),
                        'cperf-baselines', variant + '.csv')


def analyze_results(configs, args):
    old_ws = get_workspace_for_instance(OLD_INSTANCE, args)
    process_stats = os.path.join(old_ws, 'swift/utils/process-stats-dir.py')
    references = ('head', 'baseline')
    subsets = ('counters', 'timers')

    common_args = [process_stats,
                   '--markdown', '--group-by-module',
                   '--sort-by-delta-pct', '--sort-descending']

    returncodes = []
    for config in configs:
        variant = get_variant(config, args)
        sd_old = get_stats_dir(OLD_INSTANCE, variant)
        sd_new = get_stats_dir(NEW_INSTANCE, variant)
        returncodes.append(
            common.execute(
                common_args +
                ['--output', get_table_name('head', 'counters', variant),
                 '--exclude-timers',
                 '--compare-stats-dirs', sd_old, sd_new]))
        returncodes.append(
            common.execute(
                common_args +
                ['--output', get_table_name('head', 'timers', variant),
                 '--select-stat', 'driver.*user',
                 '--compare-stats-dirs', sd_old, sd_new]))
        baseline = get_baseline_name(variant)
        if os.path.exists(baseline):
            returncodes.append(
                common.execute(
                    common_args +
                    ['--output',
                     get_table_name('baseline', 'counters', variant),
                     '--exclude-timers',
                     '--compare-to-csv-baseline', baseline, sd_new]))
            returncodes.append(
                common.execute(
                    common_args +
                    ['--output',
                     get_table_name('baseline', 'timers', variant),
                     '--select-stat', 'driver.*user',
                     '--compare-to-csv-baseline', baseline, sd_new]))

    out = args.output
    regressions = any(x != 0 for x in returncodes)
    out.write("# Summary for %s %s\n\n" % (args.swift_branch, args.suite))
    if regressions:
        out.write("**Regressions found (see below)**\n\n")
    else:
        out.write("No regressions above thresholds\n\n")

    # Write summary TOC
    for config in configs:
        variant = get_variant(config, args)
        cdesc = get_config_desc(config)
        out.write('- %s\n' % make_internal_link(cdesc))
        for reference in references:
            rdesc = get_reference_desc(reference, config)
            out.write('    - %s\n' % make_internal_link(rdesc))
            for subset in subsets:
                tdesc = get_table_desc(reference, subset, config)
                out.write('        - %s\n' % make_internal_link(tdesc))

    out.write('\n\n')

    for config in configs:
        variant = get_variant(config, args)
        cdesc = get_config_desc(config)
        out.write("\n# %s\n" % cdesc)
        for reference in references:
            rdesc = get_reference_desc(reference, config)
            out.write("\n## %s\n" % rdesc)
            for subset in subsets:
                tdesc = get_table_desc(reference, subset, config)
                out.write("\n### %s\n" % tdesc)
                table = get_table_name(reference, subset, variant)
                if os.path.exists(table):
                    if os.stat(table).st_size == 0:
                        out.write("\nNone\n")
                    else:
                        with open(table) as t:
                            out.write(t.read())
                    os.unlink(table)
                else:
                    out.write("\nNo analysis available\n")

    out.write('\n\n<details>\n')
    for config in configs:
        variant = get_variant(config, args)
        baseline = get_baseline_name(variant)
        if os.path.exists(baseline):
            out.write("Last baseline commit on %s" %
                      os.path.basename(baseline))
            out.write("\n\n<pre>\n")
            out.write(common.check_execute_output([
                'git', 'log', '--pretty=medium', '-1', baseline
            ]))
            out.write("\n</pre>\n")
        else:
            out.write("No baseline file %s found" %
                      os.path.basename(baseline))
    out.write('\n</details>\n\n')

    return regressions


def get_default_output(basename):
    if 'WORKSPACE' in os.environ:
        workspace = os.environ['WORKSPACE']
        return os.path.join(workspace, basename)
    else:
        return basename


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('swift_branch')
    parser.add_argument('--sandbox', action='store_true')
    parser.add_argument('--url', type=str,
                        default="https://github.com/apple/swift")
    parser.add_argument('--suite',
                        metavar='(smoketest|full)',
                        help='cperf suite to run',
                        default='smoketest')
    parser.add_argument("--verbose",
                        action='store_true')
    parser.add_argument('--skip-build',
                        action='store_true')
    parser.add_argument('--skip-runner',
                        action='store_true')
    parser.add_argument('--output',
                        type=argparse.FileType('wb', 0),
                        default=get_default_output('comment.md'))
    parser.add_argument('--clean',
                        action='store_true')
    parser.add_argument('--setup-workspaces-for-pr',
                        type=int, default=None)
    return parser.parse_args()


if __name__ == '__main__':
    sys.exit(main())