diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000..d6e9cfceb0f --- /dev/null +++ b/.clang-format @@ -0,0 +1,92 @@ +# This file is an example configuration for clang-format 5.0. +# +# Note that this style definition should only be understood as a hint +# for writing new code. The rules are still work-in-progress and does +# not yet exactly match the style we have in the existing code. + +# C Language specifics +Language: Cpp + +# Use tabs whenever we need to fill whitespace that spans at least from one tab +# stop to the next one. +# +# These settings are mirrored in .editorconfig. Keep them in sync. +UseTab: ForIndentation +TabWidth: 8 +IndentWidth: 8 +ContinuationIndentWidth: 8 +ColumnLimit: 80 + +AlignAfterOpenBracket: AlwaysBreak +AlignEscapedNewlines: Left +AlignTrailingComments: false + +# Allow putting parameters onto the next line +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false + +# Don't allow short braced statements to be on a single line +# if (a) not if (a) return; +# return; +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortLoopsOnASingleLine: false +AllowShortLambdasOnASingleLine: None + +# Pack as many parameters or arguments onto the same line as possible +# int myFunction(int aaaaaaaaaaaa, int bbbbbbbb, +# int cccc); +BinPackArguments: true +BinPackParameters: false + +BreakBeforeBraces: Linux +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: false +BreakStringLiterals: false + +# The number of spaces before trailing line comments (// - comments). +# This does not affect trailing block comments (/* - comments). +SpacesBeforeTrailingComments: 1 + +# Don't insert spaces in casts +# x = (int32) y; not x = ( int32 ) y; +SpacesInCStyleCastParentheses: false + +# Don't insert spaces inside container literals +# var arr = [1, 2, 3]; not var arr = [ 1, 2, 3 ]; +SpacesInContainerLiterals: false + +# Don't insert spaces after '(' or before ')' +# f(arg); not f( arg ); +SpacesInParentheses: false + +# Don't insert spaces after '[' or before ']' +# int a[5]; not int a[ 5 ]; +SpacesInSquareBrackets: false + +# Insert a space after '{' and before '}' in struct initializers +Cpp11BracedListStyle: false + +# A list of macros that should be interpreted as foreach loops instead of as +# function calls. +ForEachMacros: + - 'git_array_foreach' + - 'git_vector_foreach' + +# The maximum number of consecutive empty lines to keep. +MaxEmptyLinesToKeep: 1 + +# No empty line at the start of a block. +KeepEmptyLinesAtTheStartOfBlocks: false + +# Penalties +# This decides what order things should be done if a line is too long +PenaltyBreakAssignment: 10 +PenaltyBreakBeforeFirstCallParameter: 30 +PenaltyBreakComment: 10 +PenaltyBreakFirstLessLess: 0 +PenaltyBreakString: 10 +PenaltyExcessCharacter: 100 +PenaltyReturnTypeOnItsOwnLine: 60 + +SortIncludes: false diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..bc6344b93df --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,3 @@ +{ + "postCreateCommand": "bash .devcontainer/setup.sh" +} diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100755 index 00000000000..c328bf3b98b --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +sudo apt-get update +sudo apt-get -y --no-install-recommends install cmake + +mkdir build +cd build +cmake .. \ No newline at end of file diff --git a/.editorconfig b/.editorconfig index be59274e86f..2230fd86002 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,14 +1,21 @@ ; Check http://editorconfig.org/ for more informations -; Top-most EditorConfig file root = true -; tab indentation [*] indent_style = tab +tab_width = 8 trim_trailing_whitespace = true insert_final_newline = true -; 4-column space indentation +[*.yml] +indent_style = space +indent_size = 2 + [*.md] indent_style = space indent_size = 4 +trim_trailing_whitespace = false + +[*.py] +indent_style = space +indent_size = 4 diff --git a/.gitattributes b/.gitattributes index 176a458f94e..3788dc98358 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,4 @@ * text=auto +ci/**/*.sh text eol=lf +script/**/*.sh text eol=lf +tests/resources/** linguist-vendored diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index 1e432aeef76..717f8b93490 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -1,7 +1,12 @@ -You are opening a _bug report_ against the libgit2 project. If you have a -question about an API or usage, please ask on StackOverflow: -http://stackoverflow.com/questions/tagged/libgit2. Please fill out the -reproduction steps (below) and delete this introductory paragraph. Thanks! +You are opening a _bug report_ against the libgit2 project: we use +GitHub Issues for tracking bug reports and feature requests. If you +have a question about an API or usage, please ask on StackOverflow: +http://stackoverflow.com/questions/tagged/libgit2. If you want to +have high-level discussions about the libgit2 project itself, visit +https://github.com/libgit2/discussions. + +Otherwise, to report a bug, please fill out the reproduction steps +(below) and delete these introductory paragraphs. Thanks! ### Reproduction steps diff --git a/.github/actions/download-or-build-container/action.yml b/.github/actions/download-or-build-container/action.yml new file mode 100644 index 00000000000..9c83a9836c3 --- /dev/null +++ b/.github/actions/download-or-build-container/action.yml @@ -0,0 +1,109 @@ +# Run a build step in a container or directly on the Actions runner +name: Download or Build Container +description: Download a container from the package registry, or build it if it's not found + +inputs: + container: + description: Container name + type: string + required: true + dockerfile: + description: Dockerfile + type: string + base: + description: Container base + type: string + registry: + description: Docker registry to read and publish to + type: string + default: ghcr.io + config-path: + description: Path to Dockerfiles + type: string + github_token: + description: GitHub Token + type: string + +runs: + using: 'composite' + steps: + - name: Download container + run: | + IMAGE_NAME="${{ inputs.container }}" + DOCKERFILE_PATH="${{ inputs.dockerfile }}" + DOCKER_REGISTRY="${{ inputs.registry }}" + DOCKERFILE_ROOT="${{ inputs.config-path }}" + + if [ "${DOCKERFILE_PATH}" = "" ]; then + DOCKERFILE_PATH="${DOCKERFILE_ROOT}/${IMAGE_NAME}" + else + DOCKERFILE_PATH="${DOCKERFILE_ROOT}/${DOCKERFILE_PATH}" + fi + + GIT_WORKTREE=$(cd "${GITHUB_ACTION_PATH}" && git rev-parse --show-toplevel) + echo "::: git worktree is ${GIT_WORKTREE}" + cd "${GIT_WORKTREE}" + + DOCKER_CONTAINER="${GITHUB_REPOSITORY}/${IMAGE_NAME}" + DOCKER_REGISTRY_CONTAINER="${DOCKER_REGISTRY}/${DOCKER_CONTAINER}" + + echo "dockerfile=${DOCKERFILE_PATH}" >> $GITHUB_ENV + echo "docker-container=${DOCKER_CONTAINER}" >> $GITHUB_ENV + echo "docker-registry-container=${DOCKER_REGISTRY_CONTAINER}" >> $GITHUB_ENV + + # Identify the last git commit that touched the Dockerfiles + # Use this as a hash to identify the resulting docker containers + echo "::: dockerfile path is ${DOCKERFILE_PATH}" + + DOCKER_SHA=$(git log -1 --pretty=format:"%h" -- "${DOCKERFILE_PATH}") + echo "docker-sha=${DOCKER_SHA}" >> $GITHUB_ENV + + echo "::: docker sha is ${DOCKER_SHA}" + + DOCKER_REGISTRY_CONTAINER_SHA="${DOCKER_REGISTRY_CONTAINER}:${DOCKER_SHA}" + + echo "docker-registry-container-sha=${DOCKER_REGISTRY_CONTAINER_SHA}" >> $GITHUB_ENV + echo "docker-registry-container-latest=${DOCKER_REGISTRY_CONTAINER}:latest" >> $GITHUB_ENV + + echo "::: logging in to ${DOCKER_REGISTRY} as ${GITHUB_ACTOR}" + + exists="true" + docker login https://${DOCKER_REGISTRY} -u ${GITHUB_ACTOR} -p ${GITHUB_TOKEN} || exists="false" + + echo "::: pulling ${DOCKER_REGISTRY_CONTAINER_SHA}" + + if [ "${exists}" != "false" ]; then + docker pull ${DOCKER_REGISTRY_CONTAINER_SHA} || exists="false" + fi + + if [ "${exists}" = "true" ]; then + echo "::: docker container exists in registry" + echo "docker-container-exists=true" >> $GITHUB_ENV + else + echo "::: docker container does not exist in registry" + echo "docker-container-exists=false" >> $GITHUB_ENV + fi + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github_token }} + - name: Create container + run: | + if [ "${{ inputs.base }}" != "" ]; then + BASE_ARG="--build-arg BASE=${{ inputs.base }}" + fi + + GIT_WORKTREE=$(cd "${GITHUB_ACTION_PATH}" && git rev-parse --show-toplevel) + echo "::: git worktree is ${GIT_WORKTREE}" + cd "${GIT_WORKTREE}" + + docker build -t ${{ env.docker-registry-container-sha }} --build-arg UID=$(id -u) --build-arg GID=$(id -g) ${BASE_ARG} -f ${{ env.dockerfile }} . + docker tag ${{ env.docker-registry-container-sha }} ${{ env.docker-registry-container-latest }} + shell: bash + working-directory: source/${{ inputs.config-path }} + if: env.docker-container-exists != 'true' + - name: Publish container + run: | + docker push ${{ env.docker-registry-container-sha }} + docker push ${{ env.docker-registry-container-latest }} + shell: bash + if: env.docker-container-exists != 'true' && github.event_name != 'pull_request' diff --git a/.github/actions/run-build/action.yml b/.github/actions/run-build/action.yml new file mode 100644 index 00000000000..9afcfb11e72 --- /dev/null +++ b/.github/actions/run-build/action.yml @@ -0,0 +1,51 @@ +# Run a build step in a container or directly on the Actions runner +name: Run Build Step +description: Run a build step in a container or directly on the Actions runner + +inputs: + command: + description: Command to run + type: string + required: true + container: + description: Optional container to run in + type: string + container-version: + description: Version of the container to run + type: string + shell: + description: Shell to use + type: string + required: true + default: 'bash' + +runs: + using: 'composite' + steps: + - run: | + if [ -n "${{ inputs.container }}" ]; then + docker run \ + --rm \ + --user "$(id -u):$(id -g)" \ + -v "$(pwd)/source:/home/libgit2/source" \ + -v "$(pwd)/build:/home/libgit2/build" \ + -w /home/libgit2 \ + -e ASAN_SYMBOLIZER_PATH \ + -e CC \ + -e CFLAGS \ + -e CMAKE_GENERATOR \ + -e CMAKE_OPTIONS \ + -e GITTEST_NEGOTIATE_PASSWORD \ + -e GITTEST_FLAKY_STAT \ + -e PKG_CONFIG_PATH \ + -e SKIP_NEGOTIATE_TESTS \ + -e SKIP_SSH_TESTS \ + -e SKIP_PUSHOPTIONS_TESTS \ + -e TSAN_OPTIONS \ + -e UBSAN_OPTIONS \ + ${{ inputs.container-version }} \ + /bin/bash -c "${{ inputs.command }}" + else + ${{ inputs.command }} + fi + shell: ${{ inputs.shell != '' && inputs.shell || 'bash' }} diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000000..ac05fc6de70 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,38 @@ +changelog: + categories: + - title: New features + labels: + - feature + - title: Performance improvements + labels: + - performance + - title: Bug fixes + labels: + - bug + - title: Security fixes + labels: + - security + - title: Code cleanups + labels: + - cleanup + - title: Benchmarks + labels: + - benchmarks + - title: Build and CI improvements + labels: + - build + - title: Documentation improvements + labels: + - documentation + - title: Platform compatibility fixes + labels: + - compatibility + - title: Git compatibility fixes + labels: + - git compatibility + - title: Dependency updates + labels: + - dependency + - title: Other changes + labels: + - '*' diff --git a/.github/workflows/ab-perf.yml b/.github/workflows/ab-perf.yml new file mode 100644 index 00000000000..6efb6a21660 --- /dev/null +++ b/.github/workflows/ab-perf.yml @@ -0,0 +1,229 @@ +# A/B testing with benchmarks to compare a control branch (main) to a +# candidate branch (a pull request). +name: A/B Performance Test + +on: + workflow_dispatch: + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +jobs: + # Run our CI/CD builds. We build a matrix with the various build targets + # and their details. Then we build either in a docker container (Linux) + # or on the actual hosts (macOS, Windows). + build: + strategy: + fail-fast: false + matrix: + platform: + # All builds: core platforms + - name: "Linux (Noble, GCC, OpenSSL, libssh2)" + id: noble-gcc-openssl + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" + id: noble-clang-mbedtls + os: ubuntu-latest + container: + name: noble + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" + id: xenial-gcc-openssl + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" + id: xenial-gcc-mbedtls + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "macOS" + id: macos + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, Visual Studio, Schannel)" + id: windows-amd64-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, Visual Studio, WinHTTP)" + id: windows-x86-vs + os: windows-2022 + setup-script: win32 + build_prefix: RelWithDebInfo + env: + ARCH: x86 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, mingw, WinHTTP)" + id: windows-amd64-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: amd64 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, mingw, Schannel)" + id: windows-x86-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: x86 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DEXPERIMENTAL_SHA256=ON -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "A/B: ${{ matrix.platform.name }}" + steps: + - name: Check out control + uses: actions/checkout@v4 + with: + path: source/control + ref: main + - name: Check out candidate + uses: actions/checkout@v4 + with: + path: source/candidate + - name: Set up build environment + run: source/candidate/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/candidate/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare builds + run: | + mkdir build + mkdir build/control + mkdir build/candidate + - name: Build control + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ../../source/control/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Build candidate + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ../../source/candidate/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run control benchmarks + uses: ./source/control/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/control && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Run candidate benchmarks + uses: ./source/candidate/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build/candidate && ( ./benchmarks/libgit2/${{ matrix.platform.build_prefix }}/libgit2_benchmarks -rresults.json ) + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Organize results + run: | + mkdir results + mv ${BUILD_WORKSPACE:-.}/build/control/results.json ./results/control.json + mv ${BUILD_WORKSPACE:-.}/build/candidate/results.json ./results/candidate.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.platform.id }} + path: results + if: always() + + # Publish the results + publish: + name: Publish results + needs: [ build ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Download test results + uses: actions/download-artifact@v4 + with: + path: results + - name: Generate markdown + run: node ci/compare-benchmarks.js results results.md + - name: Update pull request + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const markdown = fs.readFileSync('results.md'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdown + }); diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000000..1ff7178e9d8 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,201 @@ +# Benchmark libgit2 against the git reference implementation. +name: Benchmark + +on: + workflow_dispatch: + inputs: + suite: + description: Benchmark suite to run + debug: + type: boolean + description: Debugging output + deploy: + type: boolean + description: Deploy the benchmark site + schedule: + - cron: '15 4 * * *' + +permissions: + contents: read + +jobs: + # Run our benchmarks. We build a matrix with the various build + # targets and their details. Unlike our CI builds, we run these + # directly on the VM instead of in containers since we do not + # need the breadth of platform diversity. + build: + # Only run scheduled workflows on the main repository; prevents people + # from using build minutes on their forks. + if: github.repository == 'libgit2/libgit2' + + strategy: + matrix: + platform: + - name: "Linux (clang, OpenSSL)" + id: linux + os: ubuntu-latest + setup-script: ubuntu + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + - name: "macOS" + id: macos + os: macos-latest + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + - name: "Windows (amd64, Visual Studio)" + id: windows + os: windows-2022 + setup-script: win32 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo + CMAKE_BUILD_OPTIONS: --config RelWithDebInfo + fail-fast: false + name: "Benchmark ${{ matrix.platform.name }}" + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + - name: Set up benchmark environment + run: source/ci/setup-${{ matrix.platform.setup-script }}-benchmark.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Clone resource repositories + run: | + # TODO: + # we need a superior way to package the benchmark resources; lfs + # is too expensive + # git lfs install + # git clone https://github.com/libgit2/benchmark-resources resources + + git clone --bare https://github.com/git/git resources/git + + # TODO: + # avoid linux temporarily; the linux blame benchmarks are simply + # too slow to use + # git clone --bare https://github.com/torvalds/linux resources/linux + - name: Build + run: | + mkdir build && cd build + ../source/ci/build.sh + shell: bash + - name: Benchmark + run: | + # TODO: + # avoid benchmark resource path currently + #export BENCHMARK_RESOURCES_PATH="$(pwd)/resources" + export BENCHMARK_GIT_PATH="$(pwd)/resources/git" + # avoid linux temporarily; the linux blame benchmarks are simply + # too slow to use + # export BENCHMARK_LINUX_PATH="$(pwd)/resources/linux" + + if [[ "$(uname -s)" == MINGW* ]]; then + GIT2_CLI="$(cygpath -w $(pwd))\\build\\RelWithDebInfo\\git2" + else + GIT2_CLI="$(pwd)/build/git2" + fi + + if [ "${{ github.event.inputs.suite }}" != "" ]; then + SUITE_FLAG="--suite ${{ github.event.inputs.suite }}" + fi + + if [ "${{ github.event.inputs.debug }}" = "true" ]; then + DEBUG_FLAG="--debug" + fi + + mkdir benchmark && cd benchmark + ../source/benchmarks/cli/benchmark.sh \ + ${SUITE_FLAG} ${DEBUG_FLAG} \ + --admin \ + --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \ + --json benchmarks.json --flamegraph --zip benchmarks.zip + shell: bash + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: benchmark-${{ matrix.platform.id }} + path: benchmark + if: always() + + # Publish the results + publish: + name: Publish results + needs: [ build ] + if: always() && github.repository == 'libgit2/libgit2' + runs-on: ubuntu-latest + steps: + - name: Check out benchmark repository + uses: actions/checkout@v4 + with: + repository: libgit2/benchmarks + path: site + fetch-depth: 0 + ssh-key: ${{ secrets.BENCHMARKS_PUBLISH_KEY }} + - name: Download test results + uses: actions/download-artifact@v4 + - name: Generate API + run: | + # Move today's benchmark run into the right place + for platform in linux macos windows; do + TIMESTAMP=$(jq .time.start < "benchmark-${platform}/benchmarks.json") + TIMESTAMP_LEN=$(echo -n ${TIMESTAMP} | wc -c | xargs) + DENOMINATOR=1 + if [ "${TIMESTAMP_LEN}" = "19" ]; then + DENOMINATOR="1000000000" + elif [ "${TIMESTAMP_LEN}" = "13" ]; then + DENOMINATOR="1000" + else + echo "unknown timestamp" + exit 1 + fi + + if [[ "$(uname -s)" == "Darwin" ]]; then + DATE=$(date -R -r $(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d") + else + DATE=$(date -d @$(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d") + fi + + # move the complete results in + mkdir -p "site/public/api/runs/${DATE}" + cp "benchmark-${platform}/benchmarks.json" "site/public/api/runs/${DATE}/${platform}.json" + + # unzip the individual results + PLATFORM_TEMP=$(mktemp -d) + unzip "benchmark-${platform}/benchmarks.zip" -d "${PLATFORM_TEMP}" + + mkdir -p "site/public/api/runs/${DATE}/${platform}" + find "${PLATFORM_TEMP}" -name \*\.svg -exec cp {} "site/public/api/runs/${DATE}/${platform}" \; + done + + (cd site && node scripts/aggregate.js) + shell: bash + + # in debug mode, don't deploy the site; only create a zip file and + # upload it for debugging + - name: Upload site + uses: actions/upload-artifact@v4 + with: + name: site + path: site + if: github.event_name == 'workflow_dispatch' + - name: Publish API + run: | + git config user.name 'Benchmark Site Generation' && + git config user.email 'libgit2@users.noreply.github.com' && + git add . && + git commit --allow-empty -m"benchmark update ${DATE}" && + git push origin main + shell: bash + working-directory: site + if: github.event_name == 'schedule' || github.event.inputs.deploy == 'true' diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml new file mode 100644 index 00000000000..b52571c1811 --- /dev/null +++ b/.github/workflows/build-containers.yml @@ -0,0 +1,74 @@ +# Generate the containers that we use for builds. +name: Build Containers + +on: + workflow_call: + +env: + docker-registry: ghcr.io + docker-config-path: source/ci/docker + +jobs: + # Build the docker container images that we will use for our Linux + # builds. This will identify the last commit to the repository that + # updated the docker images, and try to download the image tagged with + # that sha. If it does not exist, we'll do a docker build and push + # the image up to GitHub Packages for the actual CI/CD runs. We tag + # with both the sha and "latest" so that the subsequent runs need not + # know the sha. Only do this on CI builds (when the event is a "push") + # because PR builds from forks lack permission to write packages. + containers: + strategy: + matrix: + container: + - name: xenial + - name: bionic + - name: focal + - name: noble + - name: docurium + - name: bionic-x86 + dockerfile: bionic + base: multiarch/ubuntu-core:x86-bionic + qemu: true + - name: bionic-arm32 + dockerfile: bionic + base: multiarch/ubuntu-core:armhf-bionic + qemu: true + - name: bionic-arm64 + dockerfile: bionic + base: multiarch/ubuntu-core:arm64-bionic + qemu: true + - name: centos7 + - name: centos8 + - name: fedora + runs-on: ubuntu-latest + name: "Create container: ${{ matrix.container.name }}" + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + if: github.event_name != 'pull_request' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.container.qemu == true + - name: Download existing container + run: | + "${{ github.workspace }}/source/ci/getcontainer.sh" "${{ matrix.container.name }}" "${{ matrix.container.dockerfile }}" + env: + DOCKER_REGISTRY: ${{ env.docker-registry }} + GITHUB_TOKEN: ${{ secrets.github_token }} + working-directory: ${{ env.docker-config-path }} + if: github.event_name != 'pull_request' + - name: Build and publish image + run: | + if [ "${{ matrix.container.base }}" != "" ]; then + BASE_ARG="--build-arg BASE=${{ matrix.container.base }}" + fi + docker build -t ${{ env.docker-registry-container-sha }} --build-arg UID=$(id -u) --build-arg GID=$(id -g) ${BASE_ARG} -f ${{ env.dockerfile }} . + docker tag ${{ env.docker-registry-container-sha }} ${{ env.docker-registry-container-latest }} + docker push ${{ env.docker-registry-container-sha }} + docker push ${{ env.docker-registry-container-latest }} + working-directory: ${{ env.docker-config-path }} + if: github.event_name != 'pull_request' && env.docker-container-exists != 'true' diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000000..d82887d2741 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,76 @@ +# Update the www.libgit2.org reference documentation +name: Generate Documentation + +on: + push: + branches: [ main ] + release: + workflow_dispatch: + inputs: + force: + description: 'Force rebuild' + type: boolean + required: true + +concurrency: + group: documentation + +permissions: + contents: read + +jobs: + documentation: + name: "Generate documentation" + runs-on: "ubuntu-latest" + steps: + - name: Check out source repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + - name: Check out documentation repository + uses: actions/checkout@v4 + with: + repository: libgit2/www.libgit2.org + path: www + fetch-depth: 0 + ssh-key: ${{ secrets.DOCS_PUBLISH_KEY }} + - name: Prepare branches + run: | + for a in main $(git branch -r --list 'origin/maint/*' | sed -e "s/^ origin\///"); do + if [ "$(git rev-parse --abbrev-ref HEAD)" != "${a}" ]; then + git branch --track "$a" "origin/$a" + fi + done + working-directory: source + - name: Generate documentation + run: | + args="" + + if [ "${{ inputs.force }}" = "true" ]; then + args="--force" + fi + + npm install + ./generate --verbose $args ../.. ../../../www/docs + working-directory: source/script/api-docs + - name: Examine changes + run: | + if [ -n "$(git diff --name-only)" ]; then + echo "changes=true" >> $GITHUB_OUTPUT + else + echo "changes=false" >> $GITHUB_OUTPUT + fi + id: check + working-directory: www + - name: Publish documentation + run: | + DATE=$(date +"%Y-%m-%d") + + git config user.name 'Documentation Site Generator' + git config user.email 'libgit2@users.noreply.github.com' + git add . + git commit -m"Documentation update ${DATE}" + git push origin main + if: steps.check.outputs.changes == 'true' + working-directory: www diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml new file mode 100644 index 00000000000..cce959f1553 --- /dev/null +++ b/.github/workflows/experimental.yml @@ -0,0 +1,120 @@ +# Validation builds for experimental features; these shouldn't be +# required for pull request approval. +name: Experimental Features + +on: + push: + branches: [ main, maint/* ] + pull_request: + branches: [ main, maint/* ] + workflow_dispatch: + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +permissions: + contents: write + packages: write + +jobs: + # Run our CI/CD builds. We build a matrix with the various build targets + # and their details. Then we build either in a docker container (Linux) + # or on the actual hosts (macOS, Windows). + build: + strategy: + matrix: + platform: + # All builds: experimental SHA256 support + - name: "Linux (SHA256, Xenial, Clang, OpenSSL)" + id: linux-sha256 + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON -DEXPERIMENTAL_SHA256=ON + - name: "macOS (SHA256)" + id: macos-sha256 + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DEXPERIMENTAL_SHA256=ON + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (SHA256, amd64, Visual Studio)" + id: windows-sha256 + os: windows-2022 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DEXPERIMENTAL_SHA256=ON + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + # TODO: this is a temporary removal + SKIP_GITDAEMON_TESTS: true + fail-fast: false + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "Build: ${{ matrix.platform.name }}" + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + - name: Set up build environment + run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare build + run: mkdir build + - name: Build + uses: ./source/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Test + uses: ./source/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Upload test results + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-results-${{ matrix.platform.id }} + path: build/results_*.xml + + test_results: + name: Test results + needs: [ build ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Download test results + uses: actions/download-artifact@v4 + - name: Generate test summary + uses: test-summary/action@v2 + with: + paths: 'test-results-*/*.xml' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000..4ca339806a3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,258 @@ +# Continuous integration and pull request validation builds for the +# main and maintenance branches. +name: CI Build + +on: + push: + branches: [ main, maint/* ] + pull_request: + branches: [ main, maint/* ] + workflow_dispatch: + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +permissions: + contents: write + packages: write + +jobs: + # Run our CI/CD builds. We build a matrix with the various build targets + # and their details. Then we build either in a docker container (Linux) + # or on the actual hosts (macOS, Windows). + build: + strategy: + matrix: + platform: + # All builds: core platforms + - name: "Linux (Noble, GCC, OpenSSL, libssh2)" + id: noble-gcc-openssl + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" + id: noble-clang-mbedtls + os: ubuntu-latest + container: + name: noble + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser + CMAKE_GENERATOR: Ninja + - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" + id: xenial-gcc-openssl + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" + id: xenial-gcc-mbedtls + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 + - name: "macOS" + id: macos + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, Visual Studio, Schannel)" + id: windows-amd64-vs + os: windows-2022 + setup-script: win32 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, Visual Studio, WinHTTP)" + id: windows-x86-vs + os: windows-2022 + setup-script: win32 + env: + ARCH: x86 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, mingw, WinHTTP)" + id: windows-amd64-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: amd64 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, mingw, Schannel)" + id: windows-x86-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: x86 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + + # All builds: sanitizers + - name: "Sanitizer (Memory)" + id: sanitizer-memory + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + - name: "Sanitizer (Address)" + id: sanitizer-address + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=address -ggdb -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + - name: "Sanitizer (UndefinedBehavior)" + id: sanitizer-ub + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + - name: "Sanitizer (Thread)" + id: sanitizer-thread + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1 + fail-fast: false + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "Build: ${{ matrix.platform.name }}" + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + - name: Set up build environment + run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare build + run: mkdir build + - name: Build + uses: ./source/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Test + uses: ./source/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Upload test results + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-results-${{ matrix.platform.id }} + path: build/results_*.xml + + documentation: + name: Validate documentation + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Validate documentation + run: | + (cd script/api-docs && npm install) + script/api-docs/api-generator.js --validate-only --strict --deprecate-hard . + + test_results: + name: Test results + needs: [ build ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Download test results + uses: actions/download-artifact@v4 + - name: Generate test summary + uses: test-summary/action@v2 + with: + paths: 'test-results-*/*.xml' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..c3dc6a53970 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,507 @@ +# Nightly build for the main branch across multiple targets. +name: Nightly Build + +on: + workflow_dispatch: + schedule: + - cron: '15 1 * * *' + +env: + docker-registry: ghcr.io + docker-config-path: ci/docker + +permissions: + contents: read + packages: write + +jobs: + # Run our nightly builds. We build a matrix with the various build + # targets and their details. Then we build either in a docker container + # (Linux) or on the actual hosts (macOS, Windows). + build: + # Only run scheduled workflows on the main repository; prevents people + # from using build minutes on their forks. + if: github.repository == 'libgit2/libgit2' + + strategy: + matrix: + platform: + # All builds: core platforms + - name: "Linux (Noble, GCC, OpenSSL, libssh2)" + id: noble-gcc-openssl + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + - name: "Linux (Noble, Clang, mbedTLS, OpenSSH)" + id: noble-clang-mbedtls + os: ubuntu-latest + container: + name: noble + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DUSE_HTTP_PARSER=http-parser + CMAKE_GENERATOR: Ninja + - name: "Linux (Xenial, GCC, OpenSSL, OpenSSH)" + id: xenial-gcc-openssl + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=exec -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON + - name: "Linux (Xenial, Clang, mbedTLS, libssh2)" + id: xenial-gcc-mbedtls + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 + - name: "macOS" + id: macos + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "iOS" + id: ios + os: macos-14 + setup-script: ios + env: + CC: clang + CMAKE_OPTIONS: -DBUILD_TESTS=OFF -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DCMAKE_TOOLCHAIN_FILE=../ios.toolchain.cmake -DCMAKE_SYSTEM_NAME=iOS -DPLATFORM=OS64 + CMAKE_GENERATOR: Ninja + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_TESTS: true # Cannot exec iOS app on macOS + - name: "Windows (amd64, Visual Studio, Schannel)" + id: windows-amd64-vs + os: windows-2022 + setup-script: win32 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, Visual Studio, WinHTTP)" + id: windows-x86-vs + os: windows-2022 + setup-script: win32 + env: + ARCH: x86 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A Win32 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DCMAKE_PREFIX_PATH=D:\Temp\libssh2 + BUILD_PATH: C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin;D:\Temp\libssh2\bin + BUILD_TEMP: D:\Temp + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (amd64, mingw, WinHTTP)" + id: windows-amd64-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: amd64 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (x86, mingw, Schannel)" + id: windows-x86-mingw + os: windows-2022 + setup-script: mingw + env: + ARCH: x86 + CMAKE_GENERATOR: MinGW Makefiles + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DUSE_HTTPS=Schannel + BUILD_TEMP: D:\Temp + BUILD_PATH: D:\Temp\mingw32\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + + # All builds: sanitizers + - name: "Sanitizer (Memory)" + id: memorysanitizer + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_C_EXTENSIONS=ON -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + - name: "Sanitizer (UndefinedBehavior)" + id: ubsanitizer + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + - name: "Sanitizer (Thread)" + id: threadsanitizer + os: ubuntu-latest + setup-script: sanitizer + container: + name: noble + env: + CC: clang + CFLAGS: -fsanitize=thread -fno-optimize-sibling-calls -fno-omit-frame-pointer + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 + UBSAN_OPTIONS: print_stacktrace=1 + TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1 + + # Nightly builds: extended platforms + - name: "Linux (CentOS 7, OpenSSL)" + id: centos7-openssl + os: ubuntu-latest + container: + name: centos7 + env: + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + PKG_CONFIG_PATH: /usr/local/lib/pkgconfig + SKIP_NEGOTIATE_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + - name: "Linux (CentOS 7, dynamically-loaded OpenSSL)" + id: centos7-dynamicopenssl + os: ubuntu-latest + container: + name: centos7 + env: + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL-Dynamic -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + PKG_CONFIG_PATH: /usr/local/lib/pkgconfig + SKIP_NEGOTIATE_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + - name: "Linux (CentOS 8, OpenSSL)" + id: centos8-openssl + os: ubuntu-latest + container: + name: centos8 + env: + CMAKE_OPTIONS: -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON + PKG_CONFIG_PATH: /usr/local/lib/pkgconfig + SKIP_NEGOTIATE_TESTS: true + SKIP_SSH_TESTS: true + - name: "Linux (CentOS 8, dynamically-loaded OpenSSL)" + id: centos8-dynamicopenssl + os: ubuntu-latest + container: + name: centos8 + env: + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL-Dynamic -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON + PKG_CONFIG_PATH: /usr/local/lib/pkgconfig + SKIP_NEGOTIATE_TESTS: true + SKIP_SSH_TESTS: true + ARCH: x86 + - name: "Linux (Fedora, llhttp)" + id: fedora + os: ubuntu-latest + container: + name: fedora + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=pcre2 -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=libssh2 -DUSE_HTTP_PARSER=llhttp + - name: "Linux (Bionic, GCC, dynamically-loaded OpenSSL)" + id: bionic-gcc-dynamicopenssl + container: + name: bionic + dockerfile: bionic + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL-Dynamic -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + RUN_INVASIVE_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + os: ubuntu-latest + - name: "Linux (x86, Bionic, Clang, OpenSSL)" + id: bionic-x86-clang-openssl + container: + name: bionic-x86 + dockerfile: bionic + qemu: true + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + RUN_INVASIVE_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + os: ubuntu-latest + - name: "Linux (x86, Bionic, GCC, OpenSSL)" + id: bionic-x86-gcc-openssl + container: + name: bionic-x86 + dockerfile: bionic + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + RUN_INVASIVE_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + os: ubuntu-latest + - name: "Linux (arm32, Bionic, GCC, OpenSSL)" + id: bionic-arm32-gcc-openssl + container: + name: bionic-arm32 + dockerfile: bionic + qemu: true + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DUSE_SSH=ON + RUN_INVASIVE_TESTS: true + SKIP_PROXY_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + GITTEST_FLAKY_STAT: true + os: ubuntu-latest + - name: "Linux (arm64, Bionic, GCC, OpenSSL)" + id: bionic-arm64-gcc-openssl + container: + name: bionic-arm64 + dockerfile: bionic + qemu: true + env: + CC: gcc + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DUSE_SSH=ON + RUN_INVASIVE_TESTS: true + SKIP_PROXY_TESTS: true + SKIP_PUSHOPTIONS_TESTS: true + os: ubuntu-latest + + # Nightly builds: ensure we fallback when missing core functionality + - name: "Linux (no threads)" + id: xenial-nothreads + os: ubuntu-latest + container: + name: xenial + env: + CC: gcc + CMAKE_OPTIONS: -DTHREADSAFE=OFF -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + SKIP_PUSHOPTIONS_TESTS: true + - name: "Linux (no mmap)" + id: noble-nommap + os: ubuntu-latest + container: + name: noble + env: + CC: gcc + CFLAGS: -DNO_MMAP + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (no mmap)" + id: windows-nommap + os: windows-2022 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CFLAGS: -DNO_MMAP + CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + + # Nightly builds: extended SSL support + - name: "Linux (dynamically-loaded OpenSSL)" + id: xenial-dynamicopenssl + os: ubuntu-latest + container: + name: xenial + env: + CC: clang + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL-Dynamic -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + CMAKE_GENERATOR: Ninja + + # All builds: experimental SHA256 support + - name: "Linux (SHA256, Xenial, Clang, OpenSSL)" + id: linux-sha256 + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON + os: ubuntu-latest + - name: "macOS (SHA256)" + id: macos-sha256 + os: macos-14 + setup-script: osx + env: + CC: clang + CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DEXPERIMENTAL_SHA256=ON + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + - name: "Windows (SHA256, amd64, Visual Studio)" + id: windows-sha256 + os: windows-2022 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_OPTIONS: -A x64 -DDEBUG_LEAK_CHECKER=win32 -DDEPRECATE_HARD=ON -DEXPERIMENTAL_SHA256=ON + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + # TODO: this is a temporary removal + SKIP_GITDAEMON_TESTS: true + - name: "Linux (SHA256, Xenial, Clang, OpenSSL-FIPS)" + id: linux-sha256-fips + container: + name: xenial + env: + CC: clang + CMAKE_GENERATOR: Ninja + CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DDEBUG_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON -DUSE_SHA1=OpenSSL-FIPS -DUSE_SHA256=OpenSSL-FIPS + os: ubuntu-latest + fail-fast: false + env: ${{ matrix.platform.env }} + runs-on: ${{ matrix.platform.os }} + name: "Build ${{ matrix.platform.name }}" + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + - name: Set up build environment + run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh + shell: bash + if: matrix.platform.setup-script != '' + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static:register --reset + if: matrix.platform.container.qemu == true + - name: Set up container + uses: ./source/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: ${{ matrix.platform.container.name }} + github_token: ${{ secrets.github_token }} + dockerfile: ${{ matrix.platform.container.dockerfile }} + if: matrix.platform.container.name != '' + - name: Prepare build + run: mkdir build + - name: Build + uses: ./source/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/build.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Test + uses: ./source/.github/actions/run-build + with: + command: cd ${BUILD_WORKSPACE:-.}/build && ../source/ci/test.sh + container: ${{ matrix.platform.container.name }} + container-version: ${{ env.docker-registry-container-sha }} + shell: ${{ matrix.platform.shell }} + - name: Upload test results + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-results-${{ matrix.platform.id }} + path: build/results_*.xml + + test_results: + name: Test results + needs: [ build ] + if: ${{ always() && github.repository == 'libgit2/libgit2' }} + runs-on: ubuntu-latest + steps: + - name: Download test results + uses: actions/download-artifact@v4 + - name: Generate test summary + uses: test-summary/action@v2 + with: + paths: 'test-results-*/*.xml' + + coverity: + # Only run scheduled workflows on the main repository; prevents people + # from using build minutes on their forks. + if: github.repository == 'libgit2/libgit2' + + name: Coverity + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + path: source + fetch-depth: 0 + - name: Set up container + uses: ./source/.github/actions/download-or-build-container + with: + registry: ${{ env.docker-registry }} + config-path: ${{ env.docker-config-path }} + container: xenial + github_token: ${{ secrets.github_token }} + if: matrix.platform.container.name != '' + - name: Run Coverity + run: source/ci/coverity.sh + env: + COVERITY_TOKEN: ${{ secrets.coverity_token }} + + codeql: + # Only run scheduled workflows on the main repository; prevents people + # from using build minutes on their forks. + if: github.repository == 'libgit2/libgit2' + + permissions: + actions: read + contents: read + security-events: write + + name: CodeQL + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: 'cpp' + + - name: Build + run: | + mkdir build + cd build + cmake .. -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON + cmake --build . + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.gitignore b/.gitignore index 1ef1ec73006..1b482f038af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,33 +1,8 @@ -/tests/clar.suite -/tests/clar.suite.rule -/tests/.clarcache -/apidocs -/trash-*.exe -/libgit2.pc -/config.mak -*.o -*.a -*.exe -*.gcda -*.gcno -*.gcov -.lock-wafbuild -.waf* -build/ -build-amiga/ -tests/tmp/ -msvc/Debug/ -msvc/Release/ -*.sln -*.suo -*.vc*proj* -*.sdf -*.opensdf -*.aps -*.cmake -!cmake/Modules/*.cmake +/build/ .DS_Store *~ .*.swp -tags -mkmf.log +/tags +CMakeSettings.json +.vs +.idea diff --git a/.mailmap b/.mailmap index 8479cf6c4f7..0b16a7e1f1a 100644 --- a/.mailmap +++ b/.mailmap @@ -16,7 +16,7 @@ Xavier L. Sascha Cunz Authmillenon Authmillenon -Edward Thomson -Edward Thomson +Edward Thomson +Edward Thomson J. David Ibáñez Russell Belfer diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index af38252ce51..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Travis-CI Build for libgit2 -# see travis-ci.org for details - -language: c - -os: - - linux - - osx - -compiler: - - gcc - - clang - -# Settings to try -env: - global: - - secure: "YnhS+8n6B+uoyaYfaJ3Lei7cSJqHDPiKJCKFIF2c87YDfmCvAJke8QtE7IzjYDs7UFkTCM4ox+ph2bERUrxZbSCyEkHdjIZpKuMJfYWja/jgMqTMxdyOH9y8JLFbZsSXDIXDwqBlC6vVyl1fP90M35wuWcNTs6tctfVWVofEFbs=" - - GITTEST_INVASIVE_FS_SIZE=1 - matrix: - - OPTIONS="-DTHREADSAFE=ON -DCMAKE_BUILD_TYPE=Release" - - OPTIONS="-DTHREADSAFE=OFF -DBUILD_EXAMPLES=ON" - -addons: - apt: - packages: - - cmake - - libssh2-1-dev - - openssh-client - - openssh-server - - valgrind - -sudo: false - -matrix: - fast_finish: true - exclude: - - os: osx - compiler: gcc - include: - - compiler: gcc - env: COVERITY=1 - os: linux - - compiler: gcc - env: - - VALGRIND=1 - OPTIONS="-DBUILD_CLAR=ON -DBUILD_EXAMPLES=OFF -DDEBUG_POOL=ON -DCMAKE_BUILD_TYPE=Debug" - os: linux - allow_failures: - - env: COVERITY=1 - -install: - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then ./script/install-deps-${TRAVIS_OS_NAME}.sh; fi - -# Run the Build script and tests -script: - - script/cibuild.sh - -# Run Tests -after_success: - - if [ "$TRAVIS_OS_NAME" = "linux" -a -n "$VALGRIND" ]; then valgrind --leak-check=full --show-reachable=yes --suppressions=./libgit2_clar.supp _build/libgit2_clar -ionline; fi - -# Only watch the development and master branches -branches: - only: - - master - - /^maint.*/ - -# Notify development list when needed -notifications: - irc: - channels: - - irc.freenode.net#libgit2 - on_success: change - on_failure: always - use_notice: true - skip_join: true - campfire: - on_success: always - on_failure: always - rooms: - - secure: "sH0dpPWMirbEe7AvLddZ2yOp8rzHalGmv0bYL/LIhVw3JDI589HCYckeLMSB\n3e/FeXw4bn0EqXWEXijVa4ijbilVY6d8oprdqMdWHEodng4KvY5vID3iZSGT\nxylhahO1XHmRynKQLOAvxlc93IlpVW38vQfby8giIY1nkpspb2w=" diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000000..62d4ec949b1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,27 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(gdb) Launch", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/libgit2_tests", + "args": [], + "stopAtEntry": false, + "cwd": "${fileDirname}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000000..5dabb895af6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "sarif-viewer.connectToGithubCodeScanning": "off" +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000000..24b4d745b31 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,27 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Build", + "type": "shell", + "command": "cd build && cmake --build . --parallel", + "group": "build", + "presentation": { + "reveal": "always", + "panel": "new" + } + }, + { + "label": "Run Tests", + "type": "shell", + "command": "build/libgit2_tests -v", + "group": "test", + "presentation": { + "reveal": "always", + "panel": "new" + } + } + ] + } diff --git a/AUTHORS b/AUTHORS index 61e2113ec92..f6164ae4f63 100644 --- a/AUTHORS +++ b/AUTHORS @@ -22,6 +22,8 @@ Dmitry Kakurin Dmitry Kovega Emeric Fermas Emmanuel Rodriguez +Eric Myhre +Erik Aigner Florian Forster Holger Weiss Ingmar Vanhassel @@ -57,6 +59,7 @@ Romain Geissler Romain Muller Russell Belfer Sakari Jokinen +Sam Altier Samuel Charles "Sam" Day Sarath Lakshman Sascha Cunz @@ -68,9 +71,11 @@ Shawn O. Pearce Shuhei Tanuma Steve Frécinaux Sven Strickroth +Talya "kivikakk" Connor Tim Branyen Tim Clem Tim Harder Torsten Bögershausen Trent Mick +Venus Xeon-Blonde Vicent Marti diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index bcf8160c133..00000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,814 +0,0 @@ -v0.25 + 1 -------- - -### Changes or improvements - -* `GIT_MERGE_OPTIONS_INIT` now includes a setting to perform rename detection. - This aligns this structure with the default by `git_merge` and - `git_merge_trees` when `NULL` was provided for the options. - -### API additions - -* You can now set the default share mode on Windows for opening files using - `GIT_OPT_SET_WINDOWS_SHAREMODE` option with `git_libgit2_opts()`. - You can query the current share mode with `GIT_OPT_GET_WINDOWS_SHAREMODE`. - -### API removals - -### Breaking API changes - -* `clone_checkout_strategy` has been removed from - `git_submodule_update_option`. The checkout strategy used to clone will - be the same strategy specified in `checkout_opts`. - -v0.25 -------- - -### Changes or improvements - -* Fix repository discovery with `git_repository_discover` and - `git_repository_open_ext` to match git's handling of a ceiling - directory at the current directory. git only checks ceiling - directories when its search ascends to a parent directory. A ceiling - directory matching the starting directory will not prevent git from - finding a repository in the starting directory or a parent directory. - -* Do not fail when deleting remotes in the presence of broken - global configs which contain branches. - -* Support for reading and writing git index v4 files - -* Improve the performance of the revwalk and bring us closer to git's code. - -* The reference db has improved support for concurrency and returns `GIT_ELOCKED` - when an operation could not be performed due to locking. - -* Nanosecond resolution is now activated by default, following git's change to - do this. - -* We now restrict the set of ciphers we let OpenSSL use by default. - -* Users can now register their own merge drivers for use with `.gitattributes`. - The library also gained built-in support for the union merge driver. - -* The default for creating references is now to validate that the object does - exist. - -* Add `git_proxy_options` which is used by the different networking - implementations to let the caller specify the proxy settings instead of - relying on the environment variables. - -### API additions - -* You can now get the user-agent used by libgit2 using the - `GIT_OPT_GET_USER_AGENT` option with `git_libgit2_opts()`. - It is the counterpart to `GIT_OPT_SET_USER_AGENT`. - -* The `GIT_OPT_SET_SSL_CIPHERS` option for `git_libgit2_opts()` lets you specify - a custom list of ciphers to use for OpenSSL. - -* `git_commit_create_buffer()` creates a commit and writes it into a - user-provided buffer instead of writing it into the object db. Combine it with - `git_commit_create_with_signature()` in order to create a commit with a - cryptographic signature. - -* `git_blob_create_fromstream()` and - `git_blob_create_fromstream_commit()` allow you to create a blob by - writing into a stream. Useful when you do not know the final size or - want to copy the contents from another stream. - -* New flags for `git_repository_open_ext`: - - * `GIT_REPOSITORY_OPEN_NO_DOTGIT` - Do not check for a repository by - appending `/.git` to the `start_path`; only open the repository if - `start_path` itself points to the git directory. - * `GIT_REPOSITORY_OPEN_FROM_ENV` - Find and open a git repository, - respecting the environment variables used by the git command-line - tools. If set, `git_repository_open_ext` will ignore the other - flags and the `ceiling_dirs` argument, and will allow a NULL - `path` to use `GIT_DIR` or search from the current directory. The - search for a repository will respect `$GIT_CEILING_DIRECTORIES` - and `$GIT_DISCOVERY_ACROSS_FILESYSTEM`. The opened repository - will respect `$GIT_INDEX_FILE`, `$GIT_NAMESPACE`, - `$GIT_OBJECT_DIRECTORY`, and `$GIT_ALTERNATE_OBJECT_DIRECTORIES`. - In the future, this flag will also cause `git_repository_open_ext` - to respect `$GIT_WORK_TREE` and `$GIT_COMMON_DIR`; currently, - `git_repository_open_ext` with this flag will error out if either - `$GIT_WORK_TREE` or `$GIT_COMMON_DIR` is set. - -* `git_diff_from_buffer()` can create a `git_diff` object from the contents - of a git-style patch file. - -* `git_index_version()` and `git_index_set_version()` to get and set - the index version - -* `git_odb_expand_ids()` lets you check for the existence of multiple - objects at once. - -* The new `git_blob_dup()`, `git_commit_dup()`, `git_tag_dup()` and - `git_tree_dup()` functions provide type-specific wrappers for - `git_object_dup()` to reduce noise and increase type safety for callers. - -* `git_reference_dup()` lets you duplicate a reference to aid in ownership - management and cleanup. - -* `git_signature_from_buffer()` lets you create a signature from a string in the - format that appear in objects. - -* `git_tree_create_updated()` lets you create a tree based on another one - together with a list of updates. For the covered update cases, it's more - efficient than the `git_index` route. - -* `git_apply_patch()` applies hunks from a `git_patch` to a buffer. - -* `git_diff_to_buf()` lets you print an entire diff directory to a buffer, - similar to how `git_patch_to_buf()` works. - -* `git_proxy_init_options()` is added to initialize a `git_proxy_options` - structure at run-time. - -* `git_merge_driver_register()`, `git_merge_driver_unregister()` let you - register and unregister a custom merge driver to be used when `.gitattributes` - specifies it. - -* `git_merge_driver_lookup()` can be used to look up a merge driver by name. - -* `git_merge_driver_source_repo()`, `git_merge_driver_source_ancestor()`, - `git_merge_driver_source_ours()`, `git_merge_driver_source_theirs()`, - `git_merge_driver_source_file_options()` added as accessors to - `git_merge_driver_source`. - -### API removals - -* `git_blob_create_fromchunks()` has been removed in favour of - `git_blob_create_fromstream()`. - -### Breaking API changes - -* `git_packbuilder_object_count` and `git_packbuilder_written` now - return a `size_t` instead of a `uint32_t` for more thorough - compatibility with the rest of the library. - -* `git_packbuiler_progress` now provides explicitly sized `uint32_t` - values instead of `unsigned int`. - -* `git_diff_file` now includes an `id_abbrev` field that reflects the - number of nibbles set in the `id` field. - -* `git_odb_backend` now has a `freshen` function pointer. This optional - function pointer is similar to the `exists` function, but it will update - a last-used marker. For filesystem-based object databases, this updates - the timestamp of the file containing the object, to indicate "freshness". - If this is `NULL`, then it will not be called and the `exists` function - will be used instead. - -* `git_remote_connect()` now accepts `git_proxy_options` argument, and - `git_fetch_options` and `git_push_options` each have a `proxy_opts` field. - -* `git_merge_options` now provides a `default_driver` that can be used - to provide the name of a merge driver to be used to handle files changed - during a merge. - -v0.24 -------- - -### Changes or improvements - -* Custom merge drivers can now be registered, which allows callers to - configure callbacks to honor `merge=driver` configuration in - `.gitattributes`. - -* Custom filters can now be registered with wildcard attributes, for - example `filter=*`. Consumers should examine the attributes parameter - of the `check` function for details. - -* Symlinks are now followed when locking a file, which can be - necessary when multiple worktrees share a base repository. - -* You can now set your own user-agent to be sent for HTTP requests by - using the `GIT_OPT_SET_USER_AGENT` with `git_libgit2_opts()`. - -* You can set custom HTTP header fields to be sent along with requests - by passing them in the fetch and push options. - -* Tree objects are now assumed to be sorted. If a tree is not - correctly formed, it will give bad results. This is the git approach - and cuts a significant amount of time when reading the trees. - -* Filter registration is now protected against concurrent - registration. - -* Filenames which are not valid on Windows in an index no longer cause - to fail to parse it on that OS. - -* Rebases can now be performed purely in-memory, without touching the - repository's workdir. - -* When adding objects to the index, or when creating new tree or commit - objects, the inputs are validated to ensure that the dependent objects - exist and are of the correct type. This object validation can be - disabled with the GIT_OPT_ENABLE_STRICT_OBJECT_CREATION option. - -* The WinHTTP transport's handling of bad credentials now behaves like - the others, asking for credentials again. - -### API additions - -* `git_config_lock()` has been added, which allow for - transactional/atomic complex updates to the configuration, removing - the opportunity for concurrent operations and not committing any - changes until the unlock. - -* `git_diff_options` added a new callback `progress_cb` to report on the - progress of the diff as files are being compared. The documentation of - the existing callback `notify_cb` was updated to reflect that it only - gets called when new deltas are added to the diff. - -* `git_fetch_options` and `git_push_options` have gained a `custom_headers` - field to set the extra HTTP header fields to send. - -* `git_stream_register_tls()` lets you register a callback to be used - as the constructor for a TLS stream instead of the libgit2 built-in - one. - -* `git_commit_header_field()` allows you to look up a specific header - field in a commit. - -* `git_commit_extract_signature()` extracts the signature from a - commit and gives you both the signature and the signed data so you - can verify it. - -### API removals - -* No APIs were removed in this version. - -### Breaking API changes - -* The `git_merge_tree_flag_t` is now `git_merge_flag_t`. Subsequently, - its members are no longer prefixed with `GIT_MERGE_TREE_FLAG` but are - now prefixed with `GIT_MERGE_FLAG`, and the `tree_flags` field of the - `git_merge_options` structure is now named `flags`. - -* The `git_merge_file_flags_t` enum is now `git_merge_file_flag_t` for - consistency with other enum type names. - -* `git_cert` descendent types now have a proper `parent` member - -* It is the responsibility of the refdb backend to decide what to do - with the reflog on ref deletion. The file-based backend must delete - it, a database-backed one may wish to archive it. - -* `git_config_backend` has gained two entries. `lock` and `unlock` - with which to implement the transactional/atomic semantics for the - configuration backend. - -* `git_index_add` and `git_index_conflict_add()` will now use the case - as provided by the caller on case insensitive systems. Previous - versions would keep the case as it existed in the index. This does - not affect the higher-level `git_index_add_bypath` or - `git_index_add_frombuffer` functions. - -* The `notify_payload` field of `git_diff_options` was renamed to `payload` - to reflect that it's also the payload for the new progress callback. - -* The `git_config_level_t` enum has gained a higher-priority value - `GIT_CONFIG_LEVEL_PROGRAMDATA` which represent a rough Windows equivalent - to the system level configuration. - -* `git_rebase_options` now has a `merge_options` field. - -* The index no longer performs locking itself. This is not something - users of the library should have been relying on as it's not part of - the concurrency guarantees. - -* `git_remote_connect()` now takes a `custom_headers` argument to set - the extra HTTP header fields to send. - -v0.23 ------- - -### Changes or improvements - -* Patience and minimal diff drivers can now be used for merges. - -* Merges can now ignore whitespace changes. - -* Updated binary identification in CRLF filtering to avoid false positives in - UTF-8 files. - -* Rename and copy detection is enabled for small files. - -* Checkout can now handle an initial checkout of a repository, making - `GIT_CHECKOUT_SAFE_CREATE` unnecessary for users of clone. - -* The signature parameter in the ref-modifying functions has been - removed. Use `git_repository_set_ident()` and - `git_repository_ident()` to override the signature to be used. - -* The local transport now auto-scales the number of threads to use - when creating the packfile instead of sticking to one. - -* Reference renaming now uses the right id for the old value. - -* The annotated version of branch creation, HEAD detaching and reset - allow for specifying the expression from the user to be put into the - reflog. - -* `git_rebase_commit` now returns `GIT_EUNMERGED` when you attempt to - commit with unstaged changes. - -* On Mac OS X, we now use SecureTransport to provide the cryptographic - support for HTTPS connections insead of OpenSSL. - -* Checkout can now accept an index for the baseline computations via the - `baseline_index` member. - -* The configuration for fetching is no longer stored inside the - `git_remote` struct but has been moved to a `git_fetch_options`. The - remote functions now take these options or the callbacks instead of - setting them beforehand. - -* `git_submodule` instances are no longer cached or shared across - lookup. Each submodule represents the configuration at the time of - loading. - -* The index now uses diffs for `add_all()` and `update_all()` which - gives it a speed boost and closer semantics to git. - -* The ssh transport now reports the stderr output from the server as - the error message, which allows you to get the "repository not - found" messages. - -* `git_index_conflict_add()` will remove staged entries that exist for - conflicted paths. - -* The flags for a `git_diff_file` will now have the `GIT_DIFF_FLAG_EXISTS` - bit set when a file exists on that side of the diff. This is useful - for understanding whether a side of the diff exists in the presence of - a conflict. - -* The constructor for a write-stream into the odb now takes - `git_off_t` instead of `size_t` for the size of the blob, which - allows putting large files into the odb on 32-bit systems. - -* The remote's push and pull URLs now honor the url.$URL.insteadOf - configuration. This allows modifying URL prefixes to a custom - value via gitconfig. - -* `git_diff_foreach`, `git_diff_blobs`, `git_diff_blob_to_buffer`, - and `git_diff_buffers` now accept a new binary callback of type - `git_diff_binary_cb` that includes the binary diff information. - -* The race condition mitigations described in `racy-git.txt` have been - implemented. - -* If libcurl is installed, we will use it to connect to HTTP(S) - servers. - -### API additions - -* The `git_merge_options` gained a `file_flags` member. - -* Parsing and retrieving a configuration value as a path is exposed - via `git_config_parse_path()` and `git_config_get_path()` - respectively. - -* `git_repository_set_ident()` and `git_repository_ident()` serve to - set and query which identity will be used when writing to the - reflog. - -* `git_config_entry_free()` frees a config entry. - -* `git_config_get_string_buf()` provides a way to safely retrieve a - string from a non-snapshot configuration. - -* `git_annotated_commit_from_revspec()` allows to get an annotated - commit from an extended sha synatx string. - -* `git_repository_set_head_detached_from_annotated()`, - `git_branch_create_from_annotated()` and - `git_reset_from_annotated()` allow for the caller to provide an - annotated commit through which they can control what expression is - put into the reflog as the source/target. - -* `git_index_add_frombuffer()` can now create a blob from memory - buffer and add it to the index which is attached to a repository. - -* The structure `git_fetch_options` has been added to determine the - runtime configuration for fetching, such as callbacks, pruning and - autotag behaviour. It has the runtime initializer - `git_fetch_init_options()`. - -* The enum `git_fetch_prune_t` has been added, letting you specify the - pruning behaviour for a fetch. - -* A push operation will notify the caller of what updates it indends - to perform on the remote, which provides similar information to - git's pre-push hook. - -* `git_stash_apply()` can now apply a stashed state from the stash list, - placing the data into the working directory and index. - -* `git_stash_pop()` will apply a stashed state (like `git_stash_apply()`) - but will remove the stashed state after a successful application. - -* A new error code `GIT_EEOF` indicates an early EOF from the - server. This typically indicates an error with the URL or - configuration of the server, and tools can use this to show messages - about failing to communicate with the server. - -* A new error code `GIT_EINVALID` indicates that an argument to a - function is invalid, or an invalid operation was requested. - -* `git_diff_index_to_workdir()` and `git_diff_tree_to_index()` will now - produce deltas of type `GIT_DELTA_CONFLICTED` to indicate that the index - side of the delta is a conflict. - -* The `git_status` family of functions will now produce status of type - `GIT_STATUS_CONFLICTED` to indicate that a conflict exists for that file - in the index. - -* `git_index_entry_is_conflict()` is a utility function to determine if - a given index entry has a non-zero stage entry, indicating that it is - one side of a conflict. - -* It is now possible to pass a keypair via a buffer instead of a - path. For this, `GIT_CREDTYPE_SSH_MEMORY` and - `git_cred_ssh_key_memory_new()` have been added. - -* `git_filter_list_contains` will indicate whether a particular - filter will be run in the given filter list. - -* `git_commit_header_field()` has been added, which allows retrieving - the contents of an arbitrary header field. - -* `git_submodule_set_branch()` allows to set the configured branch for - a submodule. - -### API removals - -* `git_remote_save()` and `git_remote_clear_refspecs()` have been - removed. Remote's configuration is changed via the configuration - directly or through a convenience function which performs changes to - the configuration directly. - -* `git_remote_set_callbacks()`, `git_remote_get_callbacks()` and - `git_remote_set_transport()` have been removed and the remote no - longer stores this configuration. - -* `git_remote_set_fetch_refpecs()` and - `git_remote_set_push_refspecs()` have been removed. There is no - longer a way to set the base refspecs at run-time. - -* `git_submodule_save()` has been removed. The submodules are no - longer configured via the objects. - -* `git_submodule_reload_all()` has been removed as we no longer cache - submodules. - -### Breaking API changes - -* `git_smart_subtransport_cb` now has a `param` parameter. - -* The `git_merge_options` structure member `flags` has been renamed - to `tree_flags`. - -* The `git_merge_file_options` structure member `flags` is now - an unsigned int. It was previously a `git_merge_file_flags_t`. - -* `GIT_CHECKOUT_SAFE_CREATE` has been removed. Most users will generally - be able to switch to `GIT_CHECKOUT_SAFE`, but if you require missing - file handling during checkout, you may now use `GIT_CHECKOUT_SAFE | - GIT_CHECKOUT_RECREATE_MISSING`. - -* The `git_clone_options` and `git_submodule_update_options` - structures no longer have a `signature` field. - -* The following functions have removed the signature and/or log message - parameters in favour of git-emulating ones. - - * `git_branch_create()`, `git_branch_move()` - * `git_rebase_init()`, `git_rebase_abort()` - * `git_reference_symbolic_create_matching()`, - `git_reference_symbolic_create()`, `git_reference_create()`, - `git_reference_create_matching()`, - `git_reference_symbolic_set_target()`, - `git_reference_set_target()`, `git_reference_rename()` - * `git_remote_update_tips()`, `git_remote_fetch()`, `git_remote_push()` - * `git_repository_set_head()`, - `git_repository_set_head_detached()`, - `git_repository_detach_head()` - * `git_reset()` - -* `git_config_get_entry()` now gives back a ref-counted - `git_config_entry`. You must free it when you no longer need it. - -* `git_config_get_string()` will return an error if used on a - non-snapshot configuration, as there can be no guarantee that the - returned pointer is valid. - -* `git_note_default_ref()` now uses a `git_buf` to return the string, - as the string is otherwise not guaranteed to stay allocated. - -* `git_rebase_operation_current()` will return `GIT_REBASE_NO_OPERATION` - if it is called immediately after creating a rebase session but before - you have applied the first patch. - -* `git_rebase_options` now contains a `git_checkout_options` struct - that will be used for functions that modify the working directory, - namely `git_rebase_init`, `git_rebase_next` and - `git_rebase_abort`. As a result, `git_rebase_open` now also takes - a `git_rebase_options` and only the `git_rebase_init` and - `git_rebase_open` functions take a `git_rebase_options`, where they - will persist the options to subsequent `git_rebase` calls. - -* The `git_clone_options` struct now has fetch options in a - `fetch_opts` field instead of remote callbacks in - `remote_callbacks`. - -* The remote callbacks has gained a new member `push_negotiation` - which gets called before sending the update commands to the server. - -* The following functions no longer act on a remote instance but - change the repository's configuration. Their signatures have changed - accordingly: - - * `git_remote_set_url()`, `git_remote_seturl()` - * `git_remote_add_fetch()`, `git_remote_add_push()` and - * `git_remote_set_autotag()` - -* `git_remote_connect()` and `git_remote_prune()` now take a pointer - to the callbacks. - -* `git_remote_fetch()` and `git_remote_download()` now take a pointer - to fetch options which determine the runtime configuration. - -* The `git_remote_autotag_option_t` values have been changed. It has - gained a `_UNSPECIFIED` default value to specify no override for the - configured setting. - -* `git_remote_update_tips()` now takes a pointer to the callbacks as - well as a boolean whether to write `FETCH_HEAD` and the autotag - setting. - -* `git_remote_create_anonymous()` no longer takes a fetch refspec as - url-only remotes cannot have configured refspecs. - -* The `git_submodule_update_options` struct now has fetch options in - the `fetch_opts` field instead of callbacks in the - `remote_callbacks` field. - -* The following functions no longer act on a submodule instance but - change the repository's configuration. Their signatures have changed - accordingly: - - * `git_submodule_set_url()`, `git_submodule_set_ignore()`, - `git_submodule_set_update()`, - `git_submodule_set_fetch_recurse_submodules()`. - -* `git_submodule_status()` no longer takes a submodule instance but a - repsitory, a submodule name and an ignore setting. - -* The `push` function in the `git_transport` interface now takes a - pointer to the remote callbacks. - -* The `git_index_entry` struct's fields' types have been changed to - more accurately reflect what is in fact stored in the - index. Specifically, time and file size are 32 bits intead of 64, as - these values are truncated. - -* `GIT_EMERGECONFLICT` is now `GIT_ECONFLICT`, which more accurately - describes the nature of the error. - -* It is no longer allowed to call `git_buf_grow()` on buffers - borrowing the memory they point to. - -v0.22 ------- - -### Changes or improvements - -* `git_signature_new()` now requires a non-empty email address. - -* Use CommonCrypto libraries for SHA-1 calculation on Mac OS X. - -* Disable SSL compression and SSLv2 and SSLv3 ciphers in favor of TLSv1 - in OpenSSL. - -* The fetch behavior of remotes with autotag set to `GIT_REMOTE_DOWNLOAD_TAGS_ALL` - has been changed to match git 1.9.0 and later. In this mode, libgit2 now - fetches all tags in addition to whatever else needs to be fetched. - -* `git_checkout()` now handles case-changing renames correctly on - case-insensitive filesystems; for example renaming "readme" to "README". - -* The search for libssh2 is now done via pkg-config instead of a - custom search of a few directories. - -* Add support for core.protectHFS and core.protectNTFS. Add more - validation for filenames which we write such as references. - -* The local transport now generates textual progress output like - git-upload-pack does ("counting objects"). - -* `git_checkout_index()` can now check out an in-memory index that is not - necessarily the repository's index, so you may check out an index - that was produced by git_merge and friends while retaining the cached - information. - -* Remove the default timeout for receiving / sending data over HTTP using - the WinHTTP transport layer. - -* Add SPNEGO (Kerberos) authentication using GSSAPI on Unix systems. - -* Provide built-in objects for the empty blob (e69de29) and empty - tree (4b825dc) objects. - -* The index' tree cache is now filled upon read-tree and write-tree - and the cache is written to disk. - -* LF -> CRLF filter refuses to handle mixed-EOL files - -* LF -> CRLF filter now runs when * text = auto (with Git for Windows 1.9.4) - -* File unlocks are atomic again via rename. Read-only files on Windows are - made read-write if necessary. - -* Share open packfiles across repositories to share descriptors and mmaps. - -* Use a map for the treebuilder, making insertion O(1) - -* The build system now accepts an option EMBED_SSH_PATH which when set - tells it to include a copy of libssh2 at the given location. This is - enabled for MSVC. - -* Add support for refspecs with the asterisk in the middle of a - pattern. - -* Fetching now performs opportunistic updates. To achieve this, we - introduce a difference between active and passive refspecs, which - make `git_remote_download()` and `git_remote_fetch()` to take a list of - resfpecs to be the active list, similarly to how git fetch accepts a - list on the command-line. - -* The THREADSAFE option to build libgit2 with threading support has - been flipped to be on by default. - -* The remote object has learnt to prune remote-tracking branches. If - the remote is configured to do so, this will happen via - `git_remote_fetch()`. You can also call `git_remote_prune()` after - connecting or fetching to perform the prune. - - -### API additions - -* Introduce `git_buf_text_is_binary()` and `git_buf_text_contains_nul()` for - consumers to perform binary detection on a git_buf. - -* `git_branch_upstream_remote()` has been introduced to provide the - branch..remote configuration value. - -* Introduce `git_describe_commit()` and `git_describe_workdir()` to provide - a description of the current commit (and working tree, respectively) - based on the nearest tag or reference - -* Introduce `git_merge_bases()` and the `git_oidarray` type to expose all - merge bases between two commits. - -* Introduce `git_merge_bases_many()` to expose all merge bases between - multiple commits. - -* Introduce rebase functionality (using the merge algorithm only). - Introduce `git_rebase_init()` to begin a new rebase session, - `git_rebase_open()` to open an in-progress rebase session, - `git_rebase_commit()` to commit the current rebase operation, - `git_rebase_next()` to apply the next rebase operation, - `git_rebase_abort()` to abort an in-progress rebase and `git_rebase_finish()` - to complete a rebase operation. - -* Introduce `git_note_author()` and `git_note_committer()` to get the author - and committer information on a `git_note`, respectively. - -* A factory function for ssh has been added which allows to change the - path of the programs to execute for receive-pack and upload-pack on - the server, `git_transport_ssh_with_paths()`. - -* The ssh transport supports asking the remote host for accepted - credential types as well as multiple challeges using a single - connection. This requires to know which username you want to connect - as, so this introduces the USERNAME credential type which the ssh - transport will use to ask for the username. - -* The `GIT_EPEEL` error code has been introduced when we cannot peel a tag - to the requested object type; if the given object otherwise cannot be - peeled, `GIT_EINVALIDSPEC` is returned. - -* Introduce `GIT_REPOSITORY_INIT_RELATIVE_GITLINK` to use relative paths - when writing gitlinks, as is used by git core for submodules. - -* `git_remote_prune()` has been added. See above for description. - - -* Introduce reference transactions, which allow multiple references to - be locked at the same time and updates be queued. This also allows - us to safely update a reflog with arbitrary contents, as we need to - do for stash. - -### API removals - -* `git_remote_supported_url()` and `git_remote_is_valid_url()` have been - removed as they have become essentially useless with rsync-style ssh paths. - -* `git_clone_into()` and `git_clone_local_into()` have been removed from the - public API in favour of `git_clone callbacks`. - -* The option to ignore certificate errors via `git_remote_cert_check()` - is no longer present. Instead, `git_remote_callbacks` has gained a new - entry which lets the user perform their own certificate checks. - -### Breaking API changes - -* `git_cherry_pick()` is now `git_cherrypick()`. - -* The `git_submodule_update()` function was renamed to - `git_submodule_update_strategy()`. `git_submodule_update()` is now used to - provide functionalty similar to "git submodule update". - -* `git_treebuilder_create()` was renamed to `git_treebuilder_new()` to better - reflect it being a constructor rather than something which writes to - disk. - -* `git_treebuilder_new()` (was `git_treebuilder_create()`) now takes a - repository so that it can query repository configuration. - Subsequently, `git_treebuilder_write()` no longer takes a repository. - -* `git_threads_init()` and `git_threads_shutdown()` have been renamed to - `git_libgit2_init()` and `git_libgit2_shutdown()` to better explain what - their purpose is, as it's grown to be more than just about threads. - -* `git_libgit2_init()` and `git_libgit2_shutdown()` now return the number of - initializations of the library, so consumers may schedule work on the - first initialization. - -* The `git_transport_register()` function no longer takes a priority and takes - a URL scheme name (eg "http") instead of a prefix like "http://" - -* `git_index_name_entrycount()` and `git_index_reuc_entrycount()` now - return size_t instead of unsigned int. - -* The `context_lines` and `interhunk_lines` fields in `git_diff`_options are - now `uint32_t` instead of `uint16_t`. This allows to set them to `UINT_MAX`, - in effect asking for "infinite" context e.g. to iterate over all the - unmodified lines of a diff. - -* `git_status_file()` now takes an exact path. Use `git_status_list_new()` if - pathspec searching is needed. - -* `git_note_create()` has changed the position of the notes reference - name to match `git_note_remove()`. - -* Rename `git_remote_load()` to `git_remote_lookup()` to bring it in line - with the rest of the lookup functions. - -* `git_remote_rename()` now takes the repository and the remote's - current name. Accepting a remote indicates we want to change it, - which we only did partially. It is much clearer if we accept a name - and no loaded objects are changed. - -* `git_remote_delete()` now accepts the repository and the remote's name - instead of a loaded remote. - -* `git_merge_head` is now `git_annotated_commit`, to better reflect its usage - for multiple functions (including rebase) - -* The `git_clone_options` struct no longer provides the `ignore_cert_errors` or - `remote_name` members for remote customization. - - Instead, the `git_clone_options` struct has two new members, `remote_cb` and - `remote_cb_payload`, which allow the caller to completely override the remote - creation process. If needed, the caller can use this callback to give their - remote a name other than the default (origin) or disable cert checking. - - The `remote_callbacks` member has been preserved for convenience, although it - is not used when a remote creation callback is supplied. - -* The `git_clone`_options struct now provides `repository_cb` and - `repository_cb_payload` to allow the user to create a repository with - custom options. - -* The `git_push` struct to perform a push has been replaced with - `git_remote_upload()`. The refspecs and options are passed as a - function argument. `git_push_update_tips()` is now also - `git_remote_update_tips()` and the callbacks are in the same struct as - the rest. - -* The `git_remote_set_transport()` function now sets a transport factory function, - rather than a pre-existing transport instance. - -* The `git_transport` structure definition has moved into the sys/transport.h - file. - -* libgit2 no longer automatically sets the OpenSSL locking - functions. This is not something which we can know to do. A - last-resort convenience function is provided in sys/openssl.h, - `git_openssl_set_locking()` which can be used to set the locking. diff --git a/CMakeLists.txt b/CMakeLists.txt index d5bf21f546e..3e9747575a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,739 +1,149 @@ -# CMake build script for the libgit2 project +# libgit2: the cross-platform, linkable library implementation of git. +# See `README.md` for build instructions. # -# Building (out of source build): -# > mkdir build && cd build -# > cmake .. [-DSETTINGS=VALUE] -# > cmake --build . -# -# Testing: -# > ctest -V -# -# Install: -# > cmake --build . --target install +# This top-level CMakeLists.txt sets up configuration options and +# determines which subprojects to build. -PROJECT(libgit2 C) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -CMAKE_POLICY(SET CMP0015 NEW) +cmake_minimum_required(VERSION 3.5.1) -# Add find modules to the path -SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/") +project(libgit2 VERSION "1.9.0" LANGUAGES C) -INCLUDE(CheckLibraryExists) -INCLUDE(CheckFunctionExists) -INCLUDE(CheckSymbolExists) -INCLUDE(CheckStructHasMember) -INCLUDE(AddCFlagIfSupported) -INCLUDE(FindPkgConfig) +# Add find modules to the path +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") +# # Build options # -OPTION( SONAME "Set the (SO)VERSION of the target" ON ) -OPTION( BUILD_SHARED_LIBS "Build Shared Library (OFF for Static)" ON ) -OPTION( THREADSAFE "Build libgit2 as threadsafe" ON ) -OPTION( BUILD_CLAR "Build Tests using the Clar suite" ON ) -OPTION( BUILD_EXAMPLES "Build library usage example apps" OFF ) -OPTION( TAGS "Generate tags" OFF ) -OPTION( PROFILE "Generate profiling information" OFF ) -OPTION( ENABLE_TRACE "Enables tracing support" OFF ) -OPTION( LIBGIT2_FILENAME "Name of the produced binary" OFF ) - -OPTION( USE_SHA1DC "Use SHA-1 with collision detection" OFF ) -OPTION( USE_ICONV "Link with and use iconv library" OFF ) -OPTION( USE_SSH "Link with libssh to enable SSH support" ON ) -OPTION( USE_GSSAPI "Link with libgssapi for SPNEGO auth" OFF ) -OPTION( VALGRIND "Configure build for valgrind" OFF ) -OPTION( CURL "Use curl for HTTP if available" ON) -OPTION( DEBUG_POOL "Enable debug pool allocator" OFF ) - -IF(DEBUG_POOL) - ADD_DEFINITIONS(-DGIT_DEBUG_POOL) -ENDIF() - -IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - SET( USE_ICONV ON ) - FIND_PACKAGE(Security) - FIND_PACKAGE(CoreFoundation REQUIRED) -ENDIF() - -IF(MSVC) - # This option is only available when building with MSVC. By default, libgit2 - # is build using the cdecl calling convention, which is useful if you're - # writing C. However, the CLR and Win32 API both expect stdcall. - # - # If you are writing a CLR program and want to link to libgit2, you'll want - # to turn this on by invoking CMake with the "-DSTDCALL=ON" argument. - OPTION( STDCALL "Build libgit2 with the __stdcall convention" OFF ) +# Experimental features +option(EXPERIMENTAL_SHA256 "Enable experimental SHA256 support (for R&D/testing)" OFF) + +# Optional subsystems +option(BUILD_SHARED_LIBS "Build Shared Library (OFF for Static)" ON) +option(BUILD_TESTS "Build the test suite" ON) +option(BUILD_BENCHMARKS "Build the benchmark suite" OFF) +option(BUILD_CLI "Build the command-line interface" ON) +option(BUILD_EXAMPLES "Build library usage example apps" OFF) +option(BUILD_FUZZERS "Build the fuzz targets" OFF) + +# Feature enablement and backend selection + set(USE_THREADS "" CACHE STRING "Use threads for parallel processing when possible. One of ON, OFF, or a specific provider: pthreads or win32. (Defaults to ON.)") + set(USE_SSH "" CACHE STRING "Enables SSH support and optionally selects provider. One of ON, OFF, or a specific provider: libssh2 or exec. (Defaults to OFF.)") + set(USE_HTTPS "" CACHE STRING "Enable HTTPS support and optionally selects the provider. One of ON, OFF, or a specific provider: OpenSSL, OpenSSL-FIPS, OpenSSL-Dynamic, mbedTLS, SecureTransport, Schannel, or WinHTTP. (Defaults to ON.)") + set(USE_SHA1 "" CACHE STRING "Selects SHA1 provider. One of builtin, HTTPS, or a specific provider. (Defaults to builtin.)") + set(USE_SHA256 "" CACHE STRING "Selects SHA256 provider. One of Builtin, HTTPS, or a specific provider. (Defaults to HTTPS.)") + set(USE_HTTP_PARSER "" CACHE STRING "Selects HTTP Parser support: http-parser, llhttp, or builtin. (Defaults to builtin.)") + set(USE_AUTH_NTLM "" CACHE STRING "Enables NTLM authentication support. One of Builtin or win32.") + set(USE_AUTH_NEGOTIATE "" CACHE STRING "Enable Negotiate (SPNEGO) authentication support. One of GSSAPI or win32.") +# set(USE_XDIFF "" CACHE STRING "Specifies the xdiff implementation; either system or builtin.") + set(USE_REGEX "" CACHE STRING "Selects regex provider. One of regcomp_l, pcre2, pcre, regcomp, or builtin.") + set(USE_COMPRESSION "" CACHE STRING "Selects compression backend. Either builtin or zlib.") + set(USE_NSEC "" CACHE STRING "Enable nanosecond precision timestamps. One of ON, OFF, or a specific provider: mtimespec, mtim, mtime, or win32. (Defaults to ON).") + +if(APPLE) + # Currently only available on macOS for `precomposeUnicode` support + set(USE_I18N "" CACHE STRING "Enables internationalization support.") +endif() + +# Debugging options + set(DEBUG_LEAK_CHECKER "" CACHE STRING "Configure for leak checking test runs. One of valgrind, leaks, or win32. Either valgrind or leaks.") +option(USE_STANDALONE_FUZZERS "Enable standalone fuzzers (compatible with gcc)" OFF) +option(DEBUG_POOL "Enable debug pool allocator" OFF) +option(DEBUG_STRICT_ALLOC "Enable strict allocator behavior" OFF) +option(DEBUG_STRICT_OPEN "Enable path validation in open" OFF) + +# Output options +option(SONAME "Set the (SO)VERSION of the target" ON) + set(LIBGIT2_FILENAME "git2" CACHE STRING "Name of the produced binary") +option(DEPRECATE_HARD "Do not include deprecated functions in the library" OFF) + +# Compilation options +# Default to c99 on Android Studio for compatibility; c90 everywhere else +if("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") + set(CMAKE_C_STANDARD "99" CACHE STRING "The C standard to compile against") +else() + set(CMAKE_C_STANDARD "90" CACHE STRING "The C standard to compile against") +endif() +option(CMAKE_C_EXTENSIONS "Whether compiler extensions are supported" OFF) +option(ENABLE_WERROR "Enable compilation with -Werror" OFF) + +if(UNIX) + option(ENABLE_REPRODUCIBLE_BUILDS "Enable reproducible builds" OFF) +endif() + +if(MSVC) # This option must match the settings used in your program, in particular if you # are linking statically - OPTION( STATIC_CRT "Link the static CRT libraries" ON ) - - # If you want to embed a copy of libssh2 into libgit2, pass a - # path to libssh2 - OPTION( EMBED_SSH_PATH "Path to libssh2 to embed (Windows)" OFF ) - - ADD_DEFINITIONS(-D_SCL_SECURE_NO_WARNINGS) - ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE) - ADD_DEFINITIONS(-D_CRT_NONSTDC_NO_DEPRECATE) -ENDIF() - - -IF(WIN32) - # By default, libgit2 is built with WinHTTP. To use the built-in - # HTTP transport, invoke CMake with the "-DWINHTTP=OFF" argument. - OPTION( WINHTTP "Use Win32 WinHTTP routines" ON ) -ENDIF() - -IF(MSVC) - # Enable MSVC CRTDBG memory leak reporting when in debug mode. - OPTION(MSVC_CRTDBG "Enable CRTDBG memory leak reporting" OFF) -ENDIF() - -IF (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - OPTION( USE_OPENSSL "Link with and use openssl library" ON ) -ENDIF() - -CHECK_STRUCT_HAS_MEMBER ("struct stat" st_mtim "sys/types.h;sys/stat.h" - HAVE_STRUCT_STAT_ST_MTIM LANGUAGE C) -CHECK_STRUCT_HAS_MEMBER ("struct stat" st_mtimespec "sys/types.h;sys/stat.h" - HAVE_STRUCT_STAT_ST_MTIMESPEC LANGUAGE C) -CHECK_STRUCT_HAS_MEMBER("struct stat" st_mtime_nsec sys/stat.h - HAVE_STRUCT_STAT_MTIME_NSEC LANGUAGE C) - -IF (HAVE_STRUCT_STAT_ST_MTIM) - CHECK_STRUCT_HAS_MEMBER("struct stat" st_mtim.tv_nsec sys/stat.h - HAVE_STRUCT_STAT_NSEC LANGUAGE C) -ELSEIF (HAVE_STRUCT_STAT_ST_MTIMESPEC) - CHECK_STRUCT_HAS_MEMBER("struct stat" st_mtimespec.tv_nsec sys/stat.h - HAVE_STRUCT_STAT_NSEC LANGUAGE C) -ELSE () - SET( HAVE_STRUCT_STAT_NSEC ON ) -ENDIF() - -IF (HAVE_STRUCT_STAT_NSEC OR WIN32) - OPTION( USE_NSEC "Care about sub-second file mtimes and ctimes" ON ) -ENDIF() - -# This variable will contain the libraries we need to put into -# libgit2.pc's Requires.private. That is, what we're linking to or -# what someone who's statically linking us needs to link to. -SET(LIBGIT2_PC_REQUIRES "") -# This will be set later if we use the system's http-parser library or -# use iconv (OSX) and will be written to the Libs.private field in the -# pc file. -SET(LIBGIT2_PC_LIBS "") - -# Installation paths + option(STATIC_CRT "Link the static CRT libraries" ON) +endif() + +if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) +endif() + + +# Modules + +include(FeatureSummary) +include(CheckLibraryExists) +include(CheckFunctionExists) +include(CheckSymbolExists) +include(CheckStructHasMember) +include(CheckPrototypeDefinitionSafe) +include(AddCFlagIfSupported) +include(FindPkgLibraries) +include(FindThreads) +include(FindStatNsec) +include(Findfutimens) +include(GNUInstallDirs) +include(IdeSplitSources) +include(EnableWarnings) +include(DefaultCFlags) +include(ExperimentalFeatures) + + +# +# Subdirectories # -SET(BIN_INSTALL_DIR bin CACHE PATH "Where to install binaries to.") -SET(LIB_INSTALL_DIR lib CACHE PATH "Where to install libraries to.") -SET(INCLUDE_INSTALL_DIR include CACHE PATH "Where to install headers to.") - -# Set a couple variables to be substituted inside the .pc file. -# We can't just use LIB_INSTALL_DIR in the .pc file, as passing them as absolue -# or relative paths is both valid and supported by cmake. -SET (PKGCONFIG_PREFIX ${CMAKE_INSTALL_PREFIX}) - -IF(IS_ABSOLUTE ${LIB_INSTALL_DIR}) - SET (PKGCONFIG_LIBDIR ${LIB_INSTALL_DIR}) -ELSE(IS_ABSOLUTE ${LIB_INSTALL_DIR}) - SET (PKGCONFIG_LIBDIR "\${prefix}/${LIB_INSTALL_DIR}") -ENDIF (IS_ABSOLUTE ${LIB_INSTALL_DIR}) - -IF(IS_ABSOLUTE ${INCLUDE_INSTALL_DIR}) - SET (PKGCONFIG_INCLUDEDIR ${INCLUDE_INSTALL_DIR}) -ELSE(IS_ABSOLUTE ${INCLUDE_INSTALL_DIR}) - SET (PKGCONFIG_INCLUDEDIR "\${prefix}/${INCLUDE_INSTALL_DIR}") -ENDIF(IS_ABSOLUTE ${INCLUDE_INSTALL_DIR}) - -FUNCTION(TARGET_OS_LIBRARIES target) - IF(WIN32) - TARGET_LINK_LIBRARIES(${target} ws2_32) - ELSEIF(CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") - TARGET_LINK_LIBRARIES(${target} socket nsl) - LIST(APPEND LIBGIT2_PC_LIBS "-lsocket" "-lnsl") - SET(LIBGIT2_PC_LIBS ${LIBGIT2_PC_LIBS} PARENT_SCOPE) - ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Haiku") - TARGET_LINK_LIBRARIES(${target} network) - LIST(APPEND LIBGIT2_PC_LIBS "-lnetwork") - SET(LIBGIT2_PC_LIBS ${LIBGIT2_PC_LIBS} PARENT_SCOPE) - ENDIF() - CHECK_LIBRARY_EXISTS(rt clock_gettime "time.h" NEED_LIBRT) - IF(NEED_LIBRT) - TARGET_LINK_LIBRARIES(${target} rt) - LIST(APPEND LIBGIT2_PC_LIBS "-lrt") - SET(LIBGIT2_PC_LIBS ${LIBGIT2_PC_LIBS} PARENT_SCOPE) - ENDIF() - - IF(THREADSAFE) - TARGET_LINK_LIBRARIES(${target} ${CMAKE_THREAD_LIBS_INIT}) - LIST(APPEND LIBGIT2_PC_LIBS ${CMAKE_THREAD_LIBS_INIT}) - SET(LIBGIT2_PC_LIBS ${LIBGIT2_PC_LIBS} PARENT_SCOPE) - ENDIF() -ENDFUNCTION() - -# This function splits the sources files up into their appropriate -# subdirectories. This is especially useful for IDEs like Xcode and -# Visual Studio, so that you can navigate into the libgit2_clar project, -# and see the folders within the tests folder (instead of just seeing all -# source and tests in a single folder.) -FUNCTION(IDE_SPLIT_SOURCES target) - IF(MSVC_IDE OR CMAKE_GENERATOR STREQUAL Xcode) - GET_TARGET_PROPERTY(sources ${target} SOURCES) - FOREACH(source ${sources}) - IF(source MATCHES ".*/") - STRING(REPLACE ${CMAKE_CURRENT_SOURCE_DIR}/ "" rel ${source}) - IF(rel) - STRING(REGEX REPLACE "/([^/]*)$" "" rel ${rel}) - IF(rel) - STRING(REPLACE "/" "\\\\" rel ${rel}) - SOURCE_GROUP(${rel} FILES ${source}) - ENDIF() - ENDIF() - ENDIF() - ENDFOREACH() - ENDIF() -ENDFUNCTION() - -FILE(STRINGS "include/git2/version.h" GIT2_HEADER REGEX "^#define LIBGIT2_VERSION \"[^\"]*\"$") - -STRING(REGEX REPLACE "^.*LIBGIT2_VERSION \"([0-9]+).*$" "\\1" LIBGIT2_VERSION_MAJOR "${GIT2_HEADER}") -STRING(REGEX REPLACE "^.*LIBGIT2_VERSION \"[0-9]+\\.([0-9]+).*$" "\\1" LIBGIT2_VERSION_MINOR "${GIT2_HEADER}") -STRING(REGEX REPLACE "^.*LIBGIT2_VERSION \"[0-9]+\\.[0-9]+\\.([0-9]+).*$" "\\1" LIBGIT2_VERSION_REV "${GIT2_HEADER}") -SET(LIBGIT2_VERSION_STRING "${LIBGIT2_VERSION_MAJOR}.${LIBGIT2_VERSION_MINOR}.${LIBGIT2_VERSION_REV}") - -FILE(STRINGS "include/git2/version.h" GIT2_HEADER_SOVERSION REGEX "^#define LIBGIT2_SOVERSION [0-9]+$") -STRING(REGEX REPLACE "^.*LIBGIT2_SOVERSION ([0-9]+)$" "\\1" LIBGIT2_SOVERSION "${GIT2_HEADER_SOVERSION}") - -# Find required dependencies -INCLUDE_DIRECTORIES(src include) - -IF (SECURITY_FOUND) - # OS X 10.7 and older do not have some functions we use, fall back to OpenSSL there - CHECK_LIBRARY_EXISTS("${SECURITY_DIRS}" SSLCreateContext "Security/SecureTransport.h" HAVE_NEWER_SECURITY) - IF (HAVE_NEWER_SECURITY) - MESSAGE("-- Found Security ${SECURITY_DIRS}") - LIST(APPEND LIBGIT2_PC_LIBS "-framework Security") - ELSE() - MESSAGE("-- Security framework is too old, falling back to OpenSSL") - SET(SECURITY_FOUND "NO") - SET(SECURITY_DIRS "") - SET(SECURITY_DIR "") - SET(USE_OPENSSL "ON") - ENDIF() -ENDIF() - -IF (COREFOUNDATION_FOUND) - MESSAGE("-- Found CoreFoundation ${COREFOUNDATION_DIRS}") - LIST(APPEND LIBGIT2_PC_LIBS "-framework CoreFoundation") -ENDIF() - - -IF (WIN32 AND EMBED_SSH_PATH) - FILE(GLOB SRC_SSH "${EMBED_SSH_PATH}/src/*.c") - INCLUDE_DIRECTORIES("${EMBED_SSH_PATH}/include") - FILE(WRITE "${EMBED_SSH_PATH}/src/libssh2_config.h" "#define HAVE_WINCNG\n#define LIBSSH2_WINCNG\n#include \"../win32/libssh2_config.h\"") - ADD_DEFINITIONS(-DGIT_SSH) -ENDIF() - -IF (WIN32 AND WINHTTP) - ADD_DEFINITIONS(-DGIT_WINHTTP) - - # Since MinGW does not come with headers or an import library for winhttp, - # we have to include a private header and generate our own import library - IF (MINGW) - FIND_PROGRAM(DLLTOOL dlltool CMAKE_FIND_ROOT_PATH_BOTH) - IF (NOT DLLTOOL) - MESSAGE(FATAL_ERROR "Could not find dlltool command") - ENDIF () - - SET(LIBWINHTTP_PATH "${CMAKE_CURRENT_BINARY_DIR}/deps/winhttp") - FILE(MAKE_DIRECTORY ${LIBWINHTTP_PATH}) - - IF (CMAKE_SIZEOF_VOID_P EQUAL 8) - set(WINHTTP_DEF "${CMAKE_CURRENT_SOURCE_DIR}/deps/winhttp/winhttp64.def") - ELSE() - set(WINHTTP_DEF "${CMAKE_CURRENT_SOURCE_DIR}/deps/winhttp/winhttp.def") - ENDIF() - - ADD_CUSTOM_COMMAND( - OUTPUT ${LIBWINHTTP_PATH}/libwinhttp.a - COMMAND ${DLLTOOL} -d ${WINHTTP_DEF} -k -D winhttp.dll -l libwinhttp.a - DEPENDS ${WINHTTP_DEF} - WORKING_DIRECTORY ${LIBWINHTTP_PATH} - ) - - SET_SOURCE_FILES_PROPERTIES( - ${CMAKE_CURRENT_SOURCE_DIR}/src/transports/winhttp.c - PROPERTIES OBJECT_DEPENDS ${LIBWINHTTP_PATH}/libwinhttp.a - ) - - INCLUDE_DIRECTORIES(deps/winhttp) - LINK_DIRECTORIES(${LIBWINHTTP_PATH}) - ENDIF () - - LINK_LIBRARIES(winhttp rpcrt4 crypt32 ole32) - LIST(APPEND LIBGIT2_PC_LIBS "-lwinhttp" "-lrpcrt4" "-lcrypt32" "-lole32") -ELSE () - IF (CURL) - PKG_CHECK_MODULES(CURL libcurl) - ENDIF () - - IF (NOT AMIGA AND USE_OPENSSL) - FIND_PACKAGE(OpenSSL) - ENDIF () - - IF (CURL_FOUND) - ADD_DEFINITIONS(-DGIT_CURL) - INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIRS}) - LINK_DIRECTORIES(${CURL_LIBRARY_DIRS}) - LINK_LIBRARIES(${CURL_LIBRARIES}) - LIST(APPEND LIBGIT2_PC_LIBS ${CURL_LDFLAGS}) - ENDIF() -ENDIF() - -# Specify sha1 implementation -IF (USE_SHA1DC) - ADD_DEFINITIONS(-DGIT_SHA1_COLLISIONDETECT) - FILE(GLOB SRC_SHA1 src/hash/hash_collisiondetect.c src/hash/sha1dc/*) -ELSEIF (WIN32 AND NOT MINGW) - ADD_DEFINITIONS(-DGIT_SHA1_WIN32) - FILE(GLOB SRC_SHA1 src/hash/hash_win32.c) -ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - ADD_DEFINITIONS(-DGIT_SHA1_COMMON_CRYPTO) -ELSEIF (OPENSSL_FOUND) - ADD_DEFINITIONS(-DGIT_SHA1_OPENSSL) - IF (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - LIST(APPEND LIBGIT2_PC_LIBS "-lssl") - ELSE() - SET(LIBGIT2_PC_REQUIRES "${LIBGIT2_PC_REQUIRES} openssl") - ENDIF () -ELSE() - FILE(GLOB SRC_SHA1 src/hash/hash_generic.c) -ENDIF() - -# Enable tracing -IF (ENABLE_TRACE STREQUAL "ON") - ADD_DEFINITIONS(-DGIT_TRACE) -ENDIF() - -# Include POSIX regex when it is required -IF(WIN32 OR AMIGA OR CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") - INCLUDE_DIRECTORIES(deps/regex) - SET(SRC_REGEX deps/regex/regex.c) -ENDIF() - -# Optional external dependency: http-parser -FIND_PACKAGE(HTTP_Parser) -IF (HTTP_PARSER_FOUND AND HTTP_PARSER_VERSION_MAJOR EQUAL 2) - INCLUDE_DIRECTORIES(${HTTP_PARSER_INCLUDE_DIRS}) - LINK_LIBRARIES(${HTTP_PARSER_LIBRARIES}) - LIST(APPEND LIBGIT2_PC_LIBS "-lhttp_parser") -ELSE() - MESSAGE(STATUS "http-parser version 2 was not found; using bundled 3rd-party sources.") - INCLUDE_DIRECTORIES(deps/http-parser) - FILE(GLOB SRC_HTTP deps/http-parser/*.c deps/http-parser/*.h) -ENDIF() - -# Optional external dependency: zlib -FIND_PACKAGE(ZLIB) -IF (ZLIB_FOUND) - INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS}) - LINK_LIBRARIES(${ZLIB_LIBRARIES}) - IF(APPLE OR CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - LIST(APPEND LIBGIT2_PC_LIBS "-lz") - ELSE() - SET(LIBGIT2_PC_REQUIRES "${LIBGIT2_PC_REQUIRES} zlib") - ENDIF() -ELSE() - MESSAGE(STATUS "zlib was not found; using bundled 3rd-party sources." ) - INCLUDE_DIRECTORIES(deps/zlib) - ADD_DEFINITIONS(-DNO_VIZ -DSTDC -DNO_GZIP) - FILE(GLOB SRC_ZLIB deps/zlib/*.c deps/zlib/*.h) -ENDIF() - -# Optional external dependency: libssh2 -IF (USE_SSH) - PKG_CHECK_MODULES(LIBSSH2 libssh2) -ENDIF() -IF (LIBSSH2_FOUND) - ADD_DEFINITIONS(-DGIT_SSH) - INCLUDE_DIRECTORIES(${LIBSSH2_INCLUDE_DIRS}) - LINK_DIRECTORIES(${LIBSSH2_LIBRARY_DIRS}) - LIST(APPEND LIBGIT2_PC_LIBS ${LIBSSH2_LDFLAGS}) - #SET(LIBGIT2_PC_LIBS "${LIBGIT2_PC_LIBS} ${LIBSSH2_LDFLAGS}") - SET(SSH_LIBRARIES ${LIBSSH2_LIBRARIES}) - - CHECK_LIBRARY_EXISTS("${LIBSSH2_LIBRARIES}" libssh2_userauth_publickey_frommemory "${LIBSSH2_LIBRARY_DIRS}" HAVE_LIBSSH2_MEMORY_CREDENTIALS) - IF (HAVE_LIBSSH2_MEMORY_CREDENTIALS) - ADD_DEFINITIONS(-DGIT_SSH_MEMORY_CREDENTIALS) - ENDIF() -ELSE() - MESSAGE(STATUS "LIBSSH2 not found. Set CMAKE_PREFIX_PATH if it is installed outside of the default search path.") -ENDIF() - -# Optional external dependency: libgssapi -IF (USE_GSSAPI) - FIND_PACKAGE(GSSAPI) -ENDIF() -IF (GSSAPI_FOUND) - ADD_DEFINITIONS(-DGIT_GSSAPI) -ENDIF() - -# Optional external dependency: iconv -IF (USE_ICONV) - FIND_PACKAGE(Iconv) -ENDIF() -IF (ICONV_FOUND) - ADD_DEFINITIONS(-DGIT_USE_ICONV) - INCLUDE_DIRECTORIES(${ICONV_INCLUDE_DIR}) - LIST(APPEND LIBGIT2_PC_LIBS ${ICONV_LIBRARIES}) -ENDIF() - -# Platform specific compilation flags -IF (MSVC) - - STRING(REPLACE "/Zm1000" " " CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") - - # /GF - String pooling - # /MP - Parallel build - SET(CMAKE_C_FLAGS "/GF /MP /nologo ${CMAKE_C_FLAGS}") - - IF (STDCALL) - # /Gz - stdcall calling convention - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Gz") - ENDIF () - - IF (STATIC_CRT) - SET(CRT_FLAG_DEBUG "/MTd") - SET(CRT_FLAG_RELEASE "/MT") - ELSE() - SET(CRT_FLAG_DEBUG "/MDd") - SET(CRT_FLAG_RELEASE "/MD") - ENDIF() - - IF (MSVC_CRTDBG) - SET(CRT_FLAG_DEBUG "${CRT_FLAG_DEBUG} /DGIT_MSVC_CRTDBG") - SET(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES}" "Dbghelp.lib") - ENDIF() - - # /Zi - Create debugging information - # /Od - Disable optimization - # /D_DEBUG - #define _DEBUG - # /MTd - Statically link the multithreaded debug version of the CRT - # /MDd - Dynamically link the multithreaded debug version of the CRT - # /RTC1 - Run time checks - SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /Zi /Od /D_DEBUG /RTC1 ${CRT_FLAG_DEBUG}") - - # /DNDEBUG - Disables asserts - # /MT - Statically link the multithreaded release version of the CRT - # /MD - Dynamically link the multithreaded release version of the CRT - # /O2 - Optimize for speed - # /Oy - Enable frame pointer omission (FPO) (otherwise CMake will automatically turn it off) - # /GL - Link time code generation (whole program optimization) - # /Gy - Function-level linking - SET(CMAKE_C_FLAGS_RELEASE "/DNDEBUG /O2 /Oy /GL /Gy ${CRT_FLAG_RELEASE}") - - # /Oy- - Disable frame pointer omission (FPO) - SET(CMAKE_C_FLAGS_RELWITHDEBINFO "/DNDEBUG /Zi /O2 /Oy- /GL /Gy ${CRT_FLAG_RELEASE}") - - # /O1 - Optimize for size - SET(CMAKE_C_FLAGS_MINSIZEREL "/DNDEBUG /O1 /Oy /GL /Gy ${CRT_FLAG_RELEASE}") - - # /DYNAMICBASE - Address space load randomization (ASLR) - # /NXCOMPAT - Data execution prevention (DEP) - # /LARGEADDRESSAWARE - >2GB user address space on x86 - # /VERSION - Embed version information in PE header - SET(CMAKE_EXE_LINKER_FLAGS "/DYNAMICBASE /NXCOMPAT /LARGEADDRESSAWARE /VERSION:${LIBGIT2_VERSION_MAJOR}.${LIBGIT2_VERSION_MINOR}") - - # /DEBUG - Create a PDB - # /LTCG - Link time code generation (whole program optimization) - # /OPT:REF /OPT:ICF - Fold out duplicate code at link step - # /INCREMENTAL:NO - Required to use /LTCG - # /DEBUGTYPE:cv,fixup - Additional data embedded in the PDB (requires /INCREMENTAL:NO, so not on for Debug) - SET(CMAKE_EXE_LINKER_FLAGS_DEBUG "/DEBUG") - SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "/RELEASE /LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO") - SET(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "/DEBUG /RELEASE /LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:cv,fixup") - SET(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "/RELEASE /LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO") - - # Same linker settings for DLL as EXE - SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") - SET(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG}") - SET(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE}") - SET(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}") - SET(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL}") - - SET(WIN_RC "src/win32/git2.rc") - - # Precompiled headers - -ELSE () - SET(CMAKE_C_FLAGS "-D_GNU_SOURCE ${CMAKE_C_FLAGS}") - - ADD_C_FLAG_IF_SUPPORTED(-Wall) - ADD_C_FLAG_IF_SUPPORTED(-Wextra) - - IF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") - SET(CMAKE_C_FLAGS "-std=c99 -D_POSIX_C_SOURCE=200112L -D__EXTENSIONS__ -D_POSIX_PTHREAD_SEMANTICS ${CMAKE_C_FLAGS}") - ENDIF() - - IF (WIN32 AND NOT CYGWIN) - SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG") - ENDIF () - - IF (MINGW OR MSYS) # MinGW and MSYS always do PIC and complain if we tell them to - STRING(REGEX REPLACE "-fPIC" "" CMAKE_SHARED_LIBRARY_C_FLAGS "${CMAKE_SHARED_LIBRARY_C_FLAGS}") - ELSEIF (BUILD_SHARED_LIBS) - ADD_C_FLAG_IF_SUPPORTED(-fvisibility=hidden) - - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") - ENDIF () - - IF (MINGW) - # MinGW >= 3.14 uses the C99-style stdio functions - # automatically, but forks like mingw-w64 still want - # us to define this in order to use them - ADD_DEFINITIONS(-D__USE_MINGW_ANSI_STDIO=1) - ENDIF () - - ADD_C_FLAG_IF_SUPPORTED(-Wdocumentation) - ADD_C_FLAG_IF_SUPPORTED(-Wno-missing-field-initializers) - ADD_C_FLAG_IF_SUPPORTED(-Wstrict-aliasing=2) - ADD_C_FLAG_IF_SUPPORTED(-Wstrict-prototypes) - ADD_C_FLAG_IF_SUPPORTED(-Wdeclaration-after-statement) - ADD_C_FLAG_IF_SUPPORTED(-Wno-unused-const-variable) - ADD_C_FLAG_IF_SUPPORTED(-Wno-unused-function) - - IF (APPLE) # Apple deprecated OpenSSL - ADD_C_FLAG_IF_SUPPORTED(-Wno-deprecated-declarations) - ENDIF() - - IF (PROFILE) - SET(CMAKE_C_FLAGS "-pg ${CMAKE_C_FLAGS}") - SET(CMAKE_EXE_LINKER_FLAGS "-pg ${CMAKE_EXE_LINKER_FLAGS}") - ENDIF () -ENDIF() - -CHECK_SYMBOL_EXISTS(regcomp_l "regex.h;xlocale.h" HAVE_REGCOMP_L) -IF (HAVE_REGCOMP_L) - ADD_DEFINITIONS(-DHAVE_REGCOMP_L) -ENDIF () - -CHECK_FUNCTION_EXISTS(futimens HAVE_FUTIMENS) -IF (HAVE_FUTIMENS) - ADD_DEFINITIONS(-DHAVE_FUTIMENS) -ENDIF () - -CHECK_FUNCTION_EXISTS(qsort_r HAVE_QSORT_R) -IF (HAVE_QSORT_R) - ADD_DEFINITIONS(-DHAVE_QSORT_R) -ENDIF () - -CHECK_FUNCTION_EXISTS(qsort_s HAVE_QSORT_S) -IF (HAVE_QSORT_S) - ADD_DEFINITIONS(-DHAVE_QSORT_S) -ENDIF () - -IF( NOT CMAKE_CONFIGURATION_TYPES ) - # Build Debug by default - IF (NOT CMAKE_BUILD_TYPE) - SET(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) - ENDIF () -ELSE() - # Using a multi-configuration generator eg MSVC or Xcode - # that uses CMAKE_CONFIGURATION_TYPES and not CMAKE_BUILD_TYPE -ENDIF() - -IF (SECURITY_FOUND) - ADD_DEFINITIONS(-DGIT_SECURE_TRANSPORT) - INCLUDE_DIRECTORIES(${SECURITY_INCLUDE_DIR}) -ENDIF () - -IF (OPENSSL_FOUND) - ADD_DEFINITIONS(-DGIT_OPENSSL) - INCLUDE_DIRECTORIES(${OPENSSL_INCLUDE_DIR}) - SET(SSL_LIBRARIES ${OPENSSL_LIBRARIES}) -ENDIF() - - - -IF (THREADSAFE) - IF (NOT WIN32) - FIND_PACKAGE(Threads REQUIRED) - ENDIF() - - ADD_DEFINITIONS(-DGIT_THREADS) -ENDIF() - -IF (USE_NSEC) - ADD_DEFINITIONS(-DGIT_USE_NSEC) -ENDIF() - -IF (HAVE_STRUCT_STAT_ST_MTIM) - ADD_DEFINITIONS(-DGIT_USE_STAT_MTIM) -ELSEIF (HAVE_STRUCT_STAT_ST_MTIMESPEC) - ADD_DEFINITIONS(-DGIT_USE_STAT_MTIMESPEC) -ELSEIF (HAVE_STRUCT_STAT_ST_MTIME_NSEC) - ADD_DEFINITIONS(-DGIT_USE_STAT_MTIME_NSEC) -ENDIF() - -ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64) - -# Collect sourcefiles -FILE(GLOB SRC_H include/git2.h include/git2/*.h include/git2/sys/*.h) - -# On Windows use specific platform sources -IF (WIN32 AND NOT CYGWIN) - ADD_DEFINITIONS(-DWIN32 -D_WIN32_WINNT=0x0501) - FILE(GLOB SRC_OS src/win32/*.c src/win32/*.h) -ELSEIF (AMIGA) - ADD_DEFINITIONS(-DNO_ADDRINFO -DNO_READDIR_R -DNO_MMAP) -ELSE() - IF (VALGRIND) - ADD_DEFINITIONS(-DNO_MMAP) - ENDIF() - FILE(GLOB SRC_OS src/unix/*.c src/unix/*.h) -ENDIF() -FILE(GLOB SRC_GIT2 src/*.c src/*.h src/transports/*.c src/transports/*.h src/xdiff/*.c src/xdiff/*.h) - -# Determine architecture of the machine -IF (CMAKE_SIZEOF_VOID_P EQUAL 8) - ADD_DEFINITIONS(-DGIT_ARCH_64) -ELSEIF (CMAKE_SIZEOF_VOID_P EQUAL 4) - ADD_DEFINITIONS(-DGIT_ARCH_32) -ELSEIF (CMAKE_SIZEOF_VOID_P) - MESSAGE(FATAL_ERROR "Unsupported architecture (pointer size is ${CMAKE_SIZEOF_VOID_P} bytes)") -ELSE() - MESSAGE(FATAL_ERROR "Unsupported architecture (CMAKE_SIZEOF_VOID_P is unset)") -ENDIF() - -# Compile and link libgit2 -ADD_LIBRARY(git2 ${SRC_H} ${SRC_GIT2} ${SRC_OS} ${SRC_ZLIB} ${SRC_HTTP} ${SRC_REGEX} ${SRC_SSH} ${SRC_SHA1} ${WIN_RC}) -TARGET_LINK_LIBRARIES(git2 ${SECURITY_DIRS}) -TARGET_LINK_LIBRARIES(git2 ${COREFOUNDATION_DIRS}) -TARGET_LINK_LIBRARIES(git2 ${SSL_LIBRARIES}) -TARGET_LINK_LIBRARIES(git2 ${SSH_LIBRARIES}) -TARGET_LINK_LIBRARIES(git2 ${GSSAPI_LIBRARIES}) -TARGET_LINK_LIBRARIES(git2 ${ICONV_LIBRARIES}) -TARGET_OS_LIBRARIES(git2) - -# Workaround for Cmake bug #0011240 (see http://public.kitware.com/Bug/view.php?id=11240) -# Win64+MSVC+static libs = linker error -IF(MSVC AND GIT_ARCH_64 AND NOT BUILD_SHARED_LIBS) - SET_TARGET_PROPERTIES(git2 PROPERTIES STATIC_LIBRARY_FLAGS "/MACHINE:x64") -ENDIF() - -IDE_SPLIT_SOURCES(git2) - -IF (SONAME) - SET_TARGET_PROPERTIES(git2 PROPERTIES VERSION ${LIBGIT2_VERSION_STRING}) - SET_TARGET_PROPERTIES(git2 PROPERTIES SOVERSION ${LIBGIT2_SOVERSION}) - IF (LIBGIT2_FILENAME) - ADD_DEFINITIONS(-DLIBGIT2_FILENAME=\"${LIBGIT2_FILENAME}\") - SET_TARGET_PROPERTIES(git2 PROPERTIES OUTPUT_NAME ${LIBGIT2_FILENAME}) - ELSEIF (DEFINED LIBGIT2_PREFIX) - SET_TARGET_PROPERTIES(git2 PROPERTIES PREFIX "${LIBGIT2_PREFIX}") - ENDIF() -ENDIF() -STRING(REPLACE ";" " " LIBGIT2_PC_LIBS "${LIBGIT2_PC_LIBS}") -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/libgit2.pc.in ${CMAKE_CURRENT_BINARY_DIR}/libgit2.pc @ONLY) - -IF (MSVC_IDE) - # Precompiled headers - SET_TARGET_PROPERTIES(git2 PROPERTIES COMPILE_FLAGS "/Yuprecompiled.h /FIprecompiled.h") - SET_SOURCE_FILES_PROPERTIES(src/win32/precompiled.c COMPILE_FLAGS "/Ycprecompiled.h") -ENDIF () - -# Install -INSTALL(TARGETS git2 - RUNTIME DESTINATION ${BIN_INSTALL_DIR} - LIBRARY DESTINATION ${LIB_INSTALL_DIR} - ARCHIVE DESTINATION ${LIB_INSTALL_DIR} -) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/libgit2.pc DESTINATION ${LIB_INSTALL_DIR}/pkgconfig ) -INSTALL(DIRECTORY include/git2 DESTINATION ${INCLUDE_INSTALL_DIR} ) -INSTALL(FILES include/git2.h DESTINATION ${INCLUDE_INSTALL_DIR} ) - -# Tests -IF (BUILD_CLAR) - FIND_PACKAGE(PythonInterp) - - IF(NOT PYTHONINTERP_FOUND) - MESSAGE(FATAL_ERROR "Could not find a python interpeter, which is needed to build the tests. " - "Make sure python is available, or pass -DBUILD_CLAR=OFF to skip building the tests") - ENDIF() - - SET(CLAR_FIXTURES "${CMAKE_CURRENT_SOURCE_DIR}/tests/resources/") - SET(CLAR_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tests") - SET(CLAR_RESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/tests/resources" CACHE PATH "Path to test resources.") - ADD_DEFINITIONS(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\") - ADD_DEFINITIONS(-DCLAR_RESOURCES=\"${TEST_RESOURCES}\") - ADD_DEFINITIONS(-DCLAR_TMPDIR=\"libgit2_tests\") - - INCLUDE_DIRECTORIES(${CLAR_PATH}) - FILE(GLOB_RECURSE SRC_TEST ${CLAR_PATH}/*/*.c ${CLAR_PATH}/*/*.h) - SET(SRC_CLAR "${CLAR_PATH}/main.c" "${CLAR_PATH}/clar_libgit2.c" "${CLAR_PATH}/clar_libgit2_trace.c" "${CLAR_PATH}/clar_libgit2_timer.c" "${CLAR_PATH}/clar.c") - - ADD_CUSTOM_COMMAND( - OUTPUT ${CLAR_PATH}/clar.suite - COMMAND ${PYTHON_EXECUTABLE} generate.py -f -xonline -xstress . - DEPENDS ${SRC_TEST} - WORKING_DIRECTORY ${CLAR_PATH} - ) - - SET_SOURCE_FILES_PROPERTIES( - ${CLAR_PATH}/clar.c - PROPERTIES OBJECT_DEPENDS ${CLAR_PATH}/clar.suite) - - ADD_EXECUTABLE(libgit2_clar ${SRC_H} ${SRC_GIT2} ${SRC_OS} ${SRC_CLAR} ${SRC_TEST} ${SRC_ZLIB} ${SRC_HTTP} ${SRC_REGEX} ${SRC_SSH} ${SRC_SHA1}) - - TARGET_LINK_LIBRARIES(libgit2_clar ${COREFOUNDATION_DIRS}) - TARGET_LINK_LIBRARIES(libgit2_clar ${SECURITY_DIRS}) - TARGET_LINK_LIBRARIES(libgit2_clar ${SSL_LIBRARIES}) - TARGET_LINK_LIBRARIES(libgit2_clar ${SSH_LIBRARIES}) - TARGET_LINK_LIBRARIES(libgit2_clar ${GSSAPI_LIBRARIES}) - TARGET_LINK_LIBRARIES(libgit2_clar ${ICONV_LIBRARIES}) - TARGET_OS_LIBRARIES(libgit2_clar) - IDE_SPLIT_SOURCES(libgit2_clar) - - IF (MSVC_IDE) - # Precompiled headers - SET_TARGET_PROPERTIES(libgit2_clar PROPERTIES COMPILE_FLAGS "/Yuprecompiled.h /FIprecompiled.h") - ENDIF () - - ENABLE_TESTING() - IF (WINHTTP OR OPENSSL_FOUND OR SECURITY_FOUND) - ADD_TEST(libgit2_clar libgit2_clar -ionline) - ELSE () - ADD_TEST(libgit2_clar libgit2_clar -v) - ENDIF () - - # Add a test target which runs the cred callback tests, to be - # called after setting the url and user - ADD_TEST(libgit2_clar-cred_callback libgit2_clar -v -sonline::clone::cred_callback) - ADD_TEST(libgit2_clar-proxy_credentials_in_url libgit2_clar -v -sonline::clone::proxy_credentials_in_url) - ADD_TEST(libgit2_clar-proxy_credentials_request libgit2_clar -v -sonline::clone::proxy_credentials_request) -ENDIF () - -IF (TAGS) - FIND_PROGRAM(CTAGS ctags) - IF (NOT CTAGS) - MESSAGE(FATAL_ERROR "Could not find ctags command") - ENDIF () - - FILE(GLOB_RECURSE SRC_ALL *.[ch]) - - ADD_CUSTOM_COMMAND( - OUTPUT tags - COMMAND ${CTAGS} -a ${SRC_ALL} - DEPENDS ${SRC_ALL} - ) - ADD_CUSTOM_TARGET( - do_tags ALL - DEPENDS tags - ) -ENDIF () - -IF (BUILD_EXAMPLES) - ADD_SUBDIRECTORY(examples) -ENDIF () + +add_subdirectory(src) + +if(BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() + +if(BUILD_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + +if(BUILD_EXAMPLES) + add_subdirectory(examples) +endif() + +if(BUILD_FUZZERS) + if((BUILD_TESTS OR BUILD_EXAMPLES) AND NOT USE_STANDALONE_FUZZERS) + message(FATAL_ERROR "Cannot build the fuzzer and the tests or examples together") + endif() + add_subdirectory(fuzzers) +endif() + + +# Export for people who use us as a dependency + +if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}") + set(LIBGIT2_DEPENDENCY_OBJECTS ${LIBGIT2_DEPENDENCY_OBJECTS} PARENT_SCOPE) + set(LIBGIT2_SYSTEM_LIBS ${LIBGIT2_SYSTEM_LIBS} PARENT_SCOPE) +endif() + + +# Summary + +feature_summary(WHAT ENABLED_FEATURES DESCRIPTION "Enabled features:") +feature_summary(WHAT DISABLED_FEATURES DESCRIPTION "Disabled features:") + +# warn for not using sha1dc + +foreach(WARNING ${WARNINGS}) + message(WARNING ${WARNING}) +endforeach() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 28a143570ef..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,161 +0,0 @@ -# Welcome to libgit2! - -We're making it easy to do interesting things with git, and we'd love to have -your help. - -## Licensing - -By contributing to libgit2, you agree to release your contribution under -the terms of the license. Except for the `examples` directory, all code -is released under the [GPL v2 with linking exception](COPYING). - -The `examples` code is governed by the -[CC0 Public Domain Dedication](examples/COPYING), so that you may copy -from them into your own application. - -## Discussion & Chat - -We hang out in the -[`#libgit2`](http://webchat.freenode.net/?channels=#libgit2)) channel on -irc.freenode.net. - -Also, feel free to open an -[Issue](https://github.com/libgit2/libgit2/issues/new) to start a discussion -about any concerns you have. We like to use Issues for that so there is an -easily accessible permanent record of the conversation. - -## Libgit2 Versions - -The `master` branch is the main branch where development happens. -Releases are tagged -(e.g. [v0.21.0](https://github.com/libgit2/libgit2/releases/tag/v0.21.0) ) -and when a critical bug fix needs to be backported, it will be done on a -`-maint` maintenance branch. - -## Reporting Bugs - -First, know which version of libgit2 your problem is in and include it in -your bug report. This can either be a tag (e.g. -[v0.17.0](https://github.com/libgit2/libgit2/releases/tag/v0.17.0)) or a -commit SHA -(e.g. [01be7863](https://github.com/libgit2/libgit2/commit/01be7863)). -Using [`git describe`](http://git-scm.com/docs/git-describe) is a -great way to tell us what version you're working with. - -If you're not running against the latest `master` branch version, -please compile and test against that to avoid re-reporting an issue that's -already been fixed. - -It's *incredibly* helpful to be able to reproduce the problem. Please -include a list of steps, a bit of code, and/or a zipped repository (if -possible). Note that some of the libgit2 developers are employees of -GitHub, so if your repository is private, find us on IRC and we'll figure -out a way to help you. - -## Pull Requests - -Our work flow is a [typical GitHub -flow](https://guides.github.com/introduction/flow/index.html), where -contributors fork the [libgit2 repository](https://github.com/libgit2/libgit2), -make their changes on branch, and submit a -[Pull Request](https://help.github.com/articles/using-pull-requests) -(a.k.a. "PR"). Pull requests should usually be targeted at the `master` -branch. - -Life will be a lot easier for you (and us) if you follow this pattern -(i.e. fork, named branch, submit PR). If you use your fork's `master` -branch directly, things can get messy. - -Please include a nice description of your changes when you submit your PR; -if we have to read the whole diff to figure out why you're contributing -in the first place, you're less likely to get feedback and have your change -merged in. - -If you are starting to work on a particular area, feel free to submit a PR -that highlights your work in progress (and note in the PR title that it's -not ready to merge). These early PRs are welcome and will help in getting -visibility for your fix, allow others to comment early on the changes and -also let others know that you are currently working on something. - -Before wrapping up a PR, you should be sure to: - -* Write tests to cover any functional changes -* Update documentation for any changed public APIs -* Add to the [`CHANGELOG.md`](CHANGELOG.md) file describing any major changes - -## Unit Tests - -We believe that our unit tests allow us to keep the quality of libgit2 -high: any new changes must not cause unit test failures, and new changes -should include unit tests that cover the bug fixes or new features. -For bug fixes, we prefer unit tests that illustrate the failure before -the change, but pass with your changes. - -In addition to new tests, please ensure that your changes do not cause -any other test failures. Running the entire test suite is helpful -before you submit a pull request. When you build libgit2, the test -suite will also be built. You can run most of the tests by simply running -the resultant `libgit2_clar` binary. If you want to run a specific -unit test, you can name it with the `-s` option. For example: - - libgit2_clar -sstatus::worktree::long_filenames - -Or you can run an entire class of tests. For example, to run all the -worktree status tests: - - libgit2_clar -sstatus::worktree - -The default test run is fairly exhaustive, but it will exclude some -unit tests by default: in particular, those that talk to network -servers and the tests that manipulate the filesystem in onerous -ways (and may need to have special privileges to run). To run the -network tests: - - libgit2_clar -ionline - -In addition, various tests may be enabled by environment variables, -like the ones that write exceptionally large repositories or manipulate -the filesystem structure in unexpected ways. These tests *may be -dangerous* to run on a normal machine and may harm your filesystem. It's -not recommended that you run these; instead, the continuous integration -servers will run these (in a sandbox). - -## Porting Code From Other Open-Source Projects - -`libgit2` is licensed under the terms of the GPL v2 with a linking -exception. Any code brought in must be compatible with those terms. - -The most common case is porting code from core Git. Git is a pure GPL -project, which means that in order to port code to this project, we need the -explicit permission of the author. Check the -[`git.git-authors`](https://github.com/libgit2/libgit2/blob/development/git.git-authors) -file for authors who have already consented. - -Other licenses have other requirements; check the license of the library -you're porting code *from* to see what you need to do. As a general rule, -MIT and BSD (3-clause) licenses are typically no problem. Apache 2.0 -license typically doesn't work due to GPL incompatibility. - -If your pull request uses code from core Git, another project, or code -from a forum / Stack Overflow, then *please* flag this in your PR and make -sure you've given proper credit to the original author in the code -snippet. - -## Style Guide - -The public API of `libgit2` is [ANSI C](http://en.wikipedia.org/wiki/ANSI_C) -(a.k.a. C89) compatible. Internally, `libgit2` is written using a portable -subset of C99 - in order to compile with GCC, Clang, MSVC, etc., we keep -local variable declarations at the tops of blocks only and avoid `//` style -comments. Additionally, `libgit2` follows some extra conventions for -function and type naming, code formatting, and testing. - -We like to keep the source code consistent and easy to read. Maintaining -this takes some discipline, but it's been more than worth it. Take a look -at the [conventions -file](https://github.com/libgit2/libgit2/blob/development/CONVENTIONS.md). - -## Starter Projects - -See our [projects -list](https://github.com/libgit2/libgit2/blob/development/PROJECTS.md). diff --git a/CONVENTIONS.md b/CONVENTIONS.md deleted file mode 100644 index ffb696a4dd4..00000000000 --- a/CONVENTIONS.md +++ /dev/null @@ -1,266 +0,0 @@ -# Libgit2 Conventions - -We like to keep the source consistent and readable. Herein are some -guidelines that should help with that. - -## External API - -We have a few rules to avoid surprising ways of calling functions and -some rules for consumers of the library to avoid stepping on each -other's toes. - - - Property accessors return the value directly (e.g. an `int` or - `const char *`) but if a function can fail, we return a `int` value - and the output parameters go first in the parameter list, followed - by the object that a function is operating on, and then any other - arguments the function may need. - - - If a function returns an object as a return value, that function is - a getter and the object's lifetime is tied to the parent - object. Objects which are returned as the first argument as a - pointer-to-pointer are owned by the caller and it is responsible - for freeing it. Strings are returned via `git_buf` in order to - allow for re-use and safe freeing. - - - Most of what libgit2 does relates to I/O so as a general rule - you should assume that any function can fail due to errors as even - getting data from the filesystem can result in all sorts of errors - and complex failure cases. - - - Paths inside the Git system are separated by a slash (0x2F). If a - function accepts a path on disk, then backslashes (0x5C) are also - accepted on Windows. - - - Do not mix allocators. If something has been allocated by libgit2, - you do not know which is the right free function in the general - case. Use the free functions provided for each object type. - -## Compatibility - -`libgit2` runs on many different platforms with many different compilers. - -The public API of `libgit2` is [ANSI C](http://en.wikipedia.org/wiki/ANSI_C) -(a.k.a. C89) compatible. - -Internally, `libgit2` is written using a portable subset of C99 - in order -to maximize compatibility (e.g. with MSVC) we avoid certain C99 -extensions. Specifically, we keep local variable declarations at the tops -of blocks only and we avoid `//` style comments. - -Also, to the greatest extent possible, we try to avoid lots of `#ifdef`s -inside the core code base. This is somewhat unavoidable, but since it can -really hamper maintainability, we keep it to a minimum. - -## Match Surrounding Code - -If there is one rule to take away from this document, it is *new code should -match the surrounding code in a way that makes it impossible to distinguish -the new from the old.* Consistency is more important to us than anyone's -personal opinion about where braces should be placed or spaces vs. tabs. - -If a section of code is being completely rewritten, it is okay to bring it -in line with the standards that are laid out here, but we will not accept -submissions that contain a large number of changes that are merely -reformatting. - -## Naming Things - -All external types and functions start with `git_` and all `#define` macros -start with `GIT_`. The `libgit2` API is mostly broken into related -functional modules each with a corresponding header. All functions in a -module should be named like `git_modulename_functioname()` -(e.g. `git_repository_open()`). - -Functions with a single output parameter should name that parameter `out`. -Multiple outputs should be named `foo_out`, `bar_out`, etc. - -Parameters of type `git_oid` should be named `id`, or `foo_id`. Calls that -return an OID should be named `git_foo_id`. - -Where a callback function is used, the function should also include a -user-supplied extra input that is a `void *` named "payload" that will be -passed through to the callback at each invocation. - -## Typedefs - -Wherever possible, use `typedef`. In some cases, if a structure is just a -collection of function pointers, the pointer types don't need to be -separately typedef'd, but loose function pointer types should be. - -## Exports - -All exported functions must be declared as: - -```c -GIT_EXTERN(result_type) git_modulename_functionname(arg_list); -``` - -## Internals - -Functions whose *modulename* is followed by two underscores, -for example `git_odb__read_packed`, are semi-private functions. -They are primarily intended for use within the library itself, -and may disappear or change their signature in a future release. - -## Parameters - -Out parameters come first. - -Whenever possible, pass argument pointers as `const`. Some structures (such -as `git_repository` and `git_index`) have mutable internal structure that -prevents this. - -Callbacks should always take a `void *` payload as their last parameter. -Callback pointers are grouped with their payloads, and typically come last -when passed as arguments: - -```c -int git_foo(git_repository *repo, git_foo_cb callback, void *payload); -``` - -## Memory Ownership - -Some APIs allocate memory which the caller is responsible for freeing; others -return a pointer into a buffer that's owned by some other object. Make this -explicit in the documentation. - -## Return codes - -Most public APIs should return an `int` error code. As is typical with most -C library functions, a zero value indicates success and a negative value -indicates failure. - -Some bindings will transform these returned error codes into exception -types, so returning a semantically appropriate error code is important. -Check -[`include/git2/errors.h`](https://github.com/libgit2/libgit2/blob/development/include/git2/errors.h) -for the return codes already defined. - -In your implementation, use `giterr_set()` to provide extended error -information to callers. - -If a `libgit2` function internally invokes another function that reports an -error, but the error is not propagated up, use `giterr_clear()` to prevent -callers from getting the wrong error message later on. - - -## Structs - -Most public types should be opaque, e.g.: - -```C -typedef struct git_odb git_odb; -``` - -...with allocation functions returning an "instance" created within -the library, and not within the application. This allows the type -to grow (or shrink) in size without rebuilding client code. - -To preserve ABI compatibility, include an `int version` field in all opaque -structures, and initialize to the latest version in the construction call. -Increment the "latest" version whenever the structure changes, and try to only -append to the end of the structure. - -## Option Structures - -If a function's parameter count is too high, it may be desirable to package -up the options in a structure. Make them transparent, include a version -field, and provide an initializer constant or constructor. Using these -structures should be this easy: - -```C -git_foo_options opts = GIT_FOO_OPTIONS_INIT; -opts.baz = BAZ_OPTION_ONE; -git_foo(&opts); -``` - -## Enumerations - -Typedef all enumerated types. If each option stands alone, use the enum -type for passing them as parameters; if they are flags to be OR'ed together, -pass them as `unsigned int` or `uint32_t` or some appropriate type. - -## Code Layout - -Try to keep lines less than 80 characters long. This is a loose -requirement, but going significantly over 80 columns is not nice. - -Use common sense to wrap most code lines; public function declarations -can use a couple of different styles: - -```c -/** All on one line is okay if it fits */ -GIT_EXTERN(int) git_foo_simple(git_oid *id); - -/** Otherwise one argument per line is a good next step */ -GIT_EXTERN(int) git_foo_id( - git_oid **out, - int a, - int b); -``` - -Indent with tabs; set your editor's tab width to 4 for best effect. - -Avoid trailing whitespace and only commit Unix-style newlines (i.e. no CRLF -in the repository - just set `core.autocrlf` to true if you are writing code -on a Windows machine). - -## Documentation - -All comments should conform to Doxygen "javadoc" style conventions for -formatting the public API documentation. Try to document every parameter, -and keep the comments up to date if you change the parameter list. - -## Public Header Template - -Use this template when creating a new public header. - -```C -#ifndef INCLUDE_git_${filename}_h__ -#define INCLUDE_git_${filename}_h__ - -#include "git/common.h" - -/** - * @file git/${filename}.h - * @brief Git some description - * @defgroup git_${filename} some description routines - * @ingroup Git - * @{ - */ -GIT_BEGIN_DECL - -/* ... definitions ... */ - -/** @} */ -GIT_END_DECL -#endif -``` - -## Inlined functions - -All inlined functions must be declared as: - -```C -GIT_INLINE(result_type) git_modulename_functionname(arg_list); -``` - -`GIT_INLINE` (or `inline`) should not be used in public headers in order -to preserve ANSI C compatibility. - -## Tests - -`libgit2` uses the [clar](https://github.com/vmg/clar) testing framework. - -All PRs should have corresponding tests. - -* If the PR fixes an existing issue, the test should fail prior to applying - the PR and succeed after applying it. -* If the PR is for new functionality, then the tests should exercise that - new functionality to a certain extent. We don't require 100% coverage - right now (although we are getting stricter over time). - -When adding new tests, we prefer if you attempt to reuse existing test data -(in `tests-clar/resources/`) if possible. If you are going to add new test -repositories, please try to strip them of unnecessary files (e.g. sample -hooks, etc). diff --git a/COPYING b/COPYING index da695ebdb47..f53de296a9c 100644 --- a/COPYING +++ b/COPYING @@ -365,7 +365,7 @@ Public License instead of this License. The bundled ZLib code is licensed under the ZLib license: -Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler + (C) 1995-2022 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -406,30 +406,35 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ---------------------------------------------------------------------- -The regex library (deps/regex/) is licensed under the GNU LGPL -(available at the end of this file). +The bundled PCRE implementation (deps/pcre/) is licensed under the BSD +license. -Definitions for data structures and routines for the regular -expression library. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -Copyright (C) 1985,1989-93,1995-98,2000,2001,2002,2003,2005,2006,2008 -Free Software Foundation, Inc. -This file is part of the GNU C Library. + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. -The GNU C Library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -The GNU C Library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. -You should have received a copy of the GNU Lesser General Public -License along with the GNU C Library; if not, write to the Free -Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -02110-1301 USA. + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- @@ -991,3 +996,296 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- + +The bundled wildmatch code was originally written by Rich $alz and is +available in the public domain. + +---------------------------------------------------------------------- + +Portions of the OpenSSL headers are included under the OpenSSL license: + +Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + +Licensed under the Apache License 2.0 (the "License"). You may not use +this file except in compliance with the License. You can obtain a copy +in the file LICENSE in the source distribution or at +https://www.openssl.org/source/license.html + +---------------------------------------------------------------------- + +The xoroshiro256** implementation is licensed in the public domain: + +Written in 2018 by David Blackman and Sebastiano Vigna (vigna@acm.org) + +To the extent possible under law, the author has dedicated all copyright +and related and neighboring rights to this software to the public domain +worldwide. This software is distributed without any warranty. + +See . + +---------------------------------------------------------------------- + +The built-in SHA256 support (src/hash/rfc6234) is taken from RFC 6234 +under the following license: + +Copyright (c) 2011 IETF Trust and the persons identified as +authors of the code. All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + +- Redistributions of source code must retain the above + copyright notice, this list of conditions and + the following disclaimer. + +- Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor + the names of specific contributors, may be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +The built-in git_fs_path_basename_r() function is based on the +Android implementation, BSD licensed: + +Copyright (C) 2008 The Android Open Source Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +---------------------------------------------------------------------- + +The bundled ntlmclient code is licensed under the MIT license: + +Copyright (c) Edward Thomson. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- + +Portions of this software derived from Team Explorer Everywhere: + +Copyright (c) Microsoft Corporation + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------------------------- + +Portions of this software derived from the LLVM Compiler Infrastructure: + +Copyright (c) 2003-2016 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +--------------------------------------------------------------------------- + +Portions of this software derived from Unicode, Inc: + +Copyright 2001-2004 Unicode, Inc. + +Disclaimer + +This source code is provided as is by Unicode, Inc. No claims are +made as to fitness for any particular purpose. No warranties of any +kind are expressed or implied. The recipient agrees to determine +applicability of information provided. If this file has been +purchased on magnetic or optical media from Unicode, Inc., the +sole remedy for any claim will be exchange of defective media +within 90 days of receipt. + +Limitations on Rights to Redistribute This Code + +Unicode, Inc. hereby grants the right to freely use the information +supplied in this file in the creation of products supporting the +Unicode Standard, and to make copies of this file in any form +for internal or external distribution as long as this notice +remains attached. + +--------------------------------------------------------------------------- + +Portions of this software derived from sheredom/utf8.h: + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +--------------------------------------------------------------------------- + +Portions of this software derived from RFC 1320: + +Copyright (C) 1990-2, RSA Data Security, Inc. All rights reserved. + +License to copy and use this software is granted provided that it +is identified as the "RSA Data Security, Inc. MD4 Message-Digest +Algorithm" in all material mentioning or referencing this software +or this function. + +License is also granted to make and use derivative works provided +that such works are identified as "derived from the RSA Data +Security, Inc. MD4 Message-Digest Algorithm" in all material +mentioning or referencing the derived work. + +RSA Data Security, Inc. makes no representations concerning either +the merchantability of this software or the suitability of this +software for any particular purpose. It is provided "as is" +without express or implied warranty of any kind. + +These notices must be retained in any copies of any part of this +documentation and/or software. + +---------------------------------------------------------------------- + +The bundled llhttp dependency is licensed under the MIT license: + +Copyright Fedor Indutny, 2018. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/FUNDING.json b/FUNDING.json new file mode 100644 index 00000000000..eb2f6310368 --- /dev/null +++ b/FUNDING.json @@ -0,0 +1,7 @@ +{ + "drips": { + "ethereum": { + "ownedBy": "0x939121dD13f796C69d0Ac4185787285518081f8D" + } + } +} diff --git a/PROJECTS.md b/PROJECTS.md deleted file mode 100644 index f53a2e12072..00000000000 --- a/PROJECTS.md +++ /dev/null @@ -1,97 +0,0 @@ -Projects For LibGit2 -==================== - -So, you want to start helping out with `libgit2`? That's fantastic! We -welcome contributions and we promise we'll try to be nice. - -This is a list of libgit2 related projects that new contributors can take -on. It includes a number of good starter projects as well as some larger -ideas that no one is actively working on. - -## Before You Start - -Please start by reading the [README.md](README.md), -[CONTRIBUTING.md](CONTRIBUTING.md), and [CONVENTIONS.md](CONVENTIONS.md) -files before diving into one of these projects. Those explain our work -flow and coding conventions to help ensure that your work will be easily -integrated into libgit2. - -Next, work through the build instructions and make sure you can clone the -repository, compile it, and run the tests successfully. That will make -sure that your development environment is set up correctly and you are -ready to start on libgit2 development. - -## Starter Projects - -These are good small projects to get started with libgit2. - -* Look at the `examples/` programs, find an existing one that mirrors a - core Git command and add a missing command-line option. There are many - gaps right now and this helps demonstrate how to use the library. Here - are some specific ideas (though there are many more): - * Fix the `examples/diff.c` implementation of the `-B` - (a.k.a. `--break-rewrites`) command line option to actually look for - the optional `[][/]` configuration values. There is an - existing comment that reads `/* TODO: parse thresholds */`. The - trick to this one will be doing it in a manner that is clean and - simple, but still handles the various cases correctly (e.g. `-B/70%` - is apparently a legal setting). - * Implement the `--log-size` option for `examples/log.c`. I think all - the data is available, you would just need to add the code into the - `print_commit()` routine (along with a way of passing the option - into that function). - * As an extension to the matching idea for `examples/log.c`, add the - `-i` option to use `strcasestr()` for matches. - * For `examples/log.c`, implement the `--first-parent` option now that - libgit2 supports it in the revwalk API. -* Pick a Git command that is not already emulated in `examples/` and write - a new example that mirrors the behavior. Examples don't have to be - perfect emulations, but should demonstrate how to use the libgit2 APIs - to get results that are similar to Git commands. This lets you (and us) - easily exercise a particular facet of the API and measure compatibility - and feature parity with core git. -* Submit a PR to clarify documentation! While we do try to document all of - the APIs, your fresh eyes on the documentation will find areas that are - confusing much more easily. - -If none of these appeal to you, take a look at our issues list to see if -there are any unresolved issues you'd like to jump in on. - -## Larger Projects - -These are ideas for larger projects mostly taken from our backlog of -[Issues](https://github.com/libgit2/libgit2/issues). Please don't dive -into one of these as a first project for libgit2 - we'd rather get to -know you first by successfully shipping your work on one of the smaller -projects above. - -Some of these projects are broken down into subprojects and/or have -some incremental steps listed towards the larger goal. Those steps -might make good smaller projects by themselves. - -* Port part of the Git test suite to run against the command line emulation - in `examples/` - * Pick a Git command that is emulated in our `examples/` area - * Extract the Git tests that exercise that command - * Convert the tests to call our emulation - * These tests could go in `examples/tests/`... -* Add hooks API to enumerate and manage hooks (not run them at this point) - * Enumeration of available hooks - * Lookup API to see which hooks have a script and get the script - * Read/write API to load a hook script and write a hook script - * Eventually, callback API to invoke a hook callback when libgit2 - executes the action in question -* Isolate logic of ignore evaluation into a standalone API -* Upgrade internal libxdiff code to latest from core Git -* Tree builder improvements: - * Extend to allow building a tree hierarchy -* Apply-patch API -* Add a patch editing API to enable "git add -p" type operations -* Textconv API to filter binary data before generating diffs (something - like the current Filter API, probably). -* Performance profiling and improvement -* Support "git replace" ref replacements -* Include conflicts in diff results and in status - * GIT_DELTA_CONFLICT for items in conflict (with multiple files) - * Appropriate flags for status -* Support sparse checkout (i.e. "core.sparsecheckout" and ".git/info/sparse-checkout") diff --git a/README.md b/README.md index 472261ceffd..08eda848569 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,147 @@ libgit2 - the Git linkable library ================================== +[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9609/badge)](https://www.bestpractices.dev/projects/9609) -[![Travis Build Status](https://secure.travis-ci.org/libgit2/libgit2.svg?branch=master)](http://travis-ci.org/libgit2/libgit2) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/xvof5b4t5480a2q3/branch/master?svg=true)](https://ci.appveyor.com/project/libgit2/libgit2/branch/master) -[![Coverity Scan Build Status](https://scan.coverity.com/projects/639/badge.svg)](https://scan.coverity.com/projects/639) +| Build Status | | +| ------------ | - | +| **main** branch builds | [![CI Build](https://github.com/libgit2/libgit2/actions/workflows/main.yml/badge.svg?branch=main&event=push)](https://github.com/libgit2/libgit2/actions/workflows/main.yml?query=event%3Apush+branch%3Amain) [![Experimental Features](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml/badge.svg?branch=main)](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml?query=event%3Apush+branch%3Amain) | +| **v1.9 branch** builds | [![CI Build](https://github.com/libgit2/libgit2/actions/workflows/main.yml/badge.svg?branch=maint%2Fv1.9&event=push)](https://github.com/libgit2/libgit2/actions/workflows/main.yml?query=event%3Apush+branch%3Amaint%2Fv1.9) [![Experimental Features](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml/badge.svg?branch=maint%2Fv1.9)](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml?query=event%3Apush+branch%3Amaint%2Fv1.9) | +| **v1.8 branch** builds | [![CI Build](https://github.com/libgit2/libgit2/actions/workflows/main.yml/badge.svg?branch=maint%2Fv1.8&event=push)](https://github.com/libgit2/libgit2/actions/workflows/main.yml?query=event%3Apush+branch%3Amaint%2Fv1.8) [![Experimental Features](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml/badge.svg?branch=maint%2Fv1.8)](https://github.com/libgit2/libgit2/actions/workflows/experimental.yml?query=event%3Apush+branch%3Amaint%2Fv1.8) | +| **Nightly** builds | [![Nightly Build](https://github.com/libgit2/libgit2/actions/workflows/nightly.yml/badge.svg?branch=main&event=schedule)](https://github.com/libgit2/libgit2/actions/workflows/nightly.yml) [![Coverity Scan Status](https://scan.coverity.com/projects/639/badge.svg)](https://scan.coverity.com/projects/639) | `libgit2` is a portable, pure C implementation of the Git core methods -provided as a re-entrant linkable library with a solid API, allowing you to -write native speed custom Git applications in any language with bindings. +provided as a linkable library with a solid API, allowing to build Git +functionality into your application. + +`libgit2` is used in a variety of places, from GUI clients to hosting +providers ("forges") and countless utilities and applications in +between. Because it's written in C, it can be made available to any +other programming language through "bindings", so you can use it in +[Ruby](https://github.com/libgit2/rugged), +[.NET](https://github.com/libgit2/libgit2sharp), +[Python](http://www.pygit2.org/), +[Node.js](http://nodegit.org), +[Rust](https://github.com/rust-lang/git2-rs), and more. + +`libgit2` is licensed under a **very permissive license** (GPLv2 with +a special Linking Exception). This means that you can link against +the library with any kind of software without making that software +fall under the GPL. Changes to libgit2 would still be covered under +its GPL license. + +Table of Contents +================= + +* [Using libgit2](#using-libgit2) +* [Quick Start](#quick-start) +* [Getting Help](#getting-help) +* [What It Can Do](#what-it-can-do) +* [Optional dependencies](#optional-dependencies) +* [Initialization](#initialization) +* [Threading](#threading) +* [Conventions](#conventions) +* [Building libgit2 - Using CMake](#building-libgit2---using-cmake) + * [Building](#building) + * [Installation](#installation) + * [Advanced Usage](#advanced-usage) + * [Compiler and linker options](#compiler-and-linker-options) + * [macOS](#macos) + * [iOS](#ios) + * [Android](#android) + * [MinGW](#mingw) +* [Language Bindings](#language-bindings) +* [How Can I Contribute?](#how-can-i-contribute) +* [License](#license) + +Using libgit2 +============= + +Most of these instructions assume that you're writing an application +in C and want to use libgit2 directly. If you're _not_ using C, +and you're writing in a different language or platform like .NET, +Node.js, or Ruby, then there is probably a +"[language binding](#language-bindings)" that you can use to take care +of the messy tasks of calling into native code. + +But if you _do_ want to use libgit2 directly - because you're building +an application in C - then you may be able use an existing binary. +There are packages for the +[vcpkg](https://github.com/Microsoft/vcpkg) and +[conan](https://conan.io/center/recipes/libgit2) +package managers. And libgit2 is available in +[Homebrew](https://formulae.brew.sh/formula/libgit2) and most Linux +distributions. + +However, these versions _may_ be outdated and we recommend using the +latest version if possible. Thankfully libgit2 is not hard to compile. + +Quick Start +=========== + +**Prerequisites** for building libgit2: + +1. [CMake](https://cmake.org/), and is recommended to be installed into + your `PATH`. +2. [Python](https://www.python.org) is used by our test framework, and + should be installed into your `PATH`. +3. C compiler: libgit2 is C90 and should compile on most compilers. + * Windows: Visual Studio is recommended + * Mac: Xcode is recommended + * Unix: gcc or clang is recommended. -`libgit2` is licensed under a **very permissive license** (GPLv2 with a special -Linking Exception). This basically means that you can link it (unmodified) -with any kind of software without having to release its source code. -Additionally, the example code has been released to the public domain (see the -[separate license](examples/COPYING) for more information). +**Build** + +1. Create a build directory beneath the libgit2 source directory, + and change into it: `mkdir build && cd build` +2. Create the cmake build environment: `cmake ..` +3. Build libgit2: `cmake --build .` + +Trouble with these steps? Read our [troubleshooting guide](docs/troubleshooting.md). +More detailed build guidance is available below. Getting Help ============ -**Join us on Slack** +**Chat with us** -Visit [slack.libgit2.org](http://slack.libgit2.org/) to sign up, then join -us in `#libgit2`. If you prefer IRC, you can also point your client to our -slack channel once you've registered. +- via IRC: join [#libgit2](https://web.libera.chat/#libgit2) on + [libera](https://libera.chat). +- via Slack: visit [slack.libgit2.org](http://slack.libgit2.org/) + to sign up, then join us in `#libgit2` **Getting Help** If you have questions about the library, please be sure to check out the -[API documentation](http://libgit2.github.com/libgit2/). If you still have +[API documentation](https://libgit2.org/libgit2/). If you still have questions, reach out to us on Slack or post a question on -[StackOverflow](http://stackoverflow.com/questions/tagged/libgit2) (with the `libgit2` tag). +[StackOverflow](http://stackoverflow.com/questions/tagged/libgit2) +(with the `libgit2` tag). **Reporting Bugs** -Please open a [GitHub Issue](https://github.com/libgit2/libgit2/issues) and -include as much information as possible. If possible, provide sample code -that illustrates the problem you're seeing. If you're seeing a bug only -on a specific repository, please provide a link to it if possible. +Please open a [GitHub Issue](https://github.com/libgit2/libgit2/issues) +and include as much information as possible. If possible, provide +sample code that illustrates the problem you're seeing. If you're +seeing a bug only on a specific repository, please provide a link to +it if possible. We ask that you not open a GitHub Issue for help, only for bug reports. +**Reporting Security Issues** + +Please have a look at SECURITY.md. + What It Can Do ============== -The goal of this library is to allow its users the ability to handle Git data in -their applications from their programming language of choice, as is used in -production for many applications including the GitHub.com site, in Plastic SCM -and also powering Microsoft's Visual Studio tools for Git. +libgit2 provides you with the ability to manage Git repositories in the +programming language of your choice. It's used in production to power many +applications including GitHub.com, Plastic SCM and Azure DevOps. -It does not aim to replace the git tool or its user-facing commands. Some APIs -resemble the plumbing commands as those align closely with the concepts of the -Git system, but most commands a user would type are out of scope for this -library to implement directly. +It does not aim to replace the git tool or its user-facing commands. Some +APIs resemble the plumbing commands as those align closely with the +concepts of the Git system, but most commands a user would type are out +of scope for this library to implement directly. The library provides: @@ -71,23 +161,31 @@ The library provides: As libgit2 is purely a consumer of the Git system, we have to adjust to changes made upstream. This has two major consequences: -* Some changes may require us to change provided interfaces. While we try to - implement functions in a generic way so that no future changes are required, - we cannot promise a completely stable API. -* As we have to keep up with changes in behavior made upstream, we may lag - behind in some areas. We usually to document these incompatibilities in our - issue tracker with the label "git change". +* Some changes may require us to change provided interfaces. While + we try to implement functions in a generic way so that no future + changes are required, we cannot promise a completely stable API. +* As we have to keep up with changes in behavior made upstream, we + may lag behind in some areas. We usually to document these + incompatibilities in our issue tracker with the label "git change". Optional dependencies ===================== -While the library provides git functionality without the need for -dependencies, it can make use of a few libraries to add to it: - -- pthreads (non-Windows) to enable threadsafe access as well as multi-threaded pack generation -- OpenSSL (non-Windows) to talk over HTTPS and provide the SHA-1 functions -- LibSSH2 to enable the SSH transport -- iconv (OSX) to handle the HFS+ path encoding peculiarities +While the library provides git functionality with very few +dependencies, some recommended dependencies are used for performance +or complete functionality. + +- Hash generation: Git uses SHA1DC (collision detecting SHA1) for + its default hash generation. SHA256 support is experimental, and + optimized support is provided by system libraries on macOS and + Windows, or by the HTTPS library on Unix systems when available. +- Threading: is provided by the system libraries on Windows, and + pthreads on Unix systems. +- HTTPS: is provided by the system libraries on macOS and Windows, + or by OpenSSL or mbedTLS on other Unix systems. +- SSH: is provided by [libssh2](https://libssh2.org/) or by invoking + [OpenSSH](https://www.openssh.com). +- Unicode: is provided by the system libraries on Windows and macOS. Initialization =============== @@ -108,23 +206,23 @@ require assistance coordinating this, simply have the worker threads call Threading ========= -See [THREADING](THREADING.md) for information +See [threading](docs/threading.md) for information Conventions =========== -See [CONVENTIONS](CONVENTIONS.md) for an overview of the external +See [conventions](docs/conventions.md) for an overview of the external and internal API/coding conventions we use. Building libgit2 - Using CMake ============================== -`libgit2` builds cleanly on most platforms without any external dependencies. -Under Unix-like systems, like Linux, \*BSD and Mac OS X, libgit2 expects `pthreads` to be available; -they should be installed by default on all systems. Under Windows, libgit2 uses the native Windows API -for threading. +Building +-------- -The `libgit2` library is built using [CMake]() (version 2.8 or newer) on all platforms. +`libgit2` builds cleanly on most platforms without any external +dependencies as a requirement. `libgit2` is built using +[CMake]() (version 2.8 or newer) on all platforms. On most systems you can build the library using the following commands @@ -132,61 +230,216 @@ On most systems you can build the library using the following commands $ cmake .. $ cmake --build . +To include the examples in the build, use `cmake -DBUILD_EXAMPLES=ON ..` +instead of `cmake ..`. The built executable for the examples can then +be found in `build/examples`, relative to the toplevel directory. + Alternatively you can point the CMake GUI tool to the CMakeLists.txt file and generate platform specific build project or IDE workspace. +If you're not familiar with CMake, [a more detailed explanation](https://preshing.com/20170511/how-to-build-a-cmake-based-project/) may be helpful. + +Advanced Options +---------------- + +You can specify a number of options to `cmake` that will change the +way `libgit2` is built. To use this, specify `-Doption=value` during +the initial `cmake` configuration. For example, to enable SHA256 +compatibility: + + $ mkdir build && cd build + $ cmake -DEXPERIMENTAL_SHA256=ON .. + $ cmake --build . + +libgit2 options: + +* `EXPERIMENTAL_SHA256=ON`: turns on SHA256 compatibility; note that + this is an API-incompatible change, hence why it is labeled + "experimental" + +Build options: + +* `BUILD_EXAMPLES=ON`: builds the suite of example code +* `BUILD_FUZZERS=ON`: builds the fuzzing suite +* `ENABLE_WERROR=ON`: build with `-Werror` or the equivalent, which turns + compiler warnings into errors in the libgit2 codebase (but not its + dependencies) + +Dependency options: + +* `USE_SSH=type`: enables SSH support and optionally selects the provider; + `type` can be set to `libssh2` or `exec` (which will execute an external + OpenSSH command). `ON` implies `libssh2`; defaults to `OFF`. +* `USE_HTTPS=type`: enables HTTPS support and optionally selects the + provider; `type` can be set to `OpenSSL`, `OpenSSL-Dynamic` (to not + link against OpenSSL, but load it dynamically), `SecureTransport`, + `Schannel` or `WinHTTP`; the default is `SecureTransport` on macOS, + `WinHTTP` on Windows, and whichever of `OpenSSL` or `mbedTLS` is + detected on other platforms. Defaults to `ON`. +* `USE_SHA1=type`: selects the SHA1 mechanism to use; `type` can be set + to `CollisionDetection`, `HTTPS` to use the system or HTTPS provider, + or one of `OpenSSL`, `OpenSSL-Dynamic`, `OpenSSL-FIPS` (to use FIPS + compliant routines in OpenSSL), `CommonCrypto`, or `Schannel`. + Defaults to `CollisionDetection`. This option is retained for + backward compatibility and should not be changed. +* `USE_SHA256=type`: selects the SHA256 mechanism to use; `type` can be + set to `HTTPS` to use the system or HTTPS driver, `builtin`, or one of + `OpenSSL`, `OpenSSL-Dynamic`, `OpenSSL-FIPS` (to use FIPS compliant + routines in OpenSSL), `CommonCrypto`, or `Schannel`. Defaults to `HTTPS`. +* `USE_GSSAPI=`: enables GSSAPI for SPNEGO authentication on + Unix. Defaults to `OFF`. +* `USE_HTTP_PARSER=type`: selects the HTTP Parser; either `http-parser` + for an external + [`http-parser`](https://github.com/nodejs/http-parser) dependency, + `llhttp` for an external [`llhttp`](https://github.com/nodejs/llhttp) + dependency, or `builtin`. Defaults to `builtin`. +* `REGEX_BACKEND=type`: selects the regular expression backend to use; + one of `regcomp_l`, `pcre2`, `pcre`, `regcomp`, or `builtin`. The + default is to use `regcomp_l` where available, PCRE if found, otherwise, + to use the builtin. +* `USE_BUNDLED_ZLIB=type`: selects the bundled zlib; either `ON` or `OFF`. + Defaults to using the system zlib if available, falling back to the + bundled zlib. + +Locating Dependencies +--------------------- + +The `libgit2` project uses `cmake` since it helps with cross-platform +projects, especially those with many dependencies. If your dependencies +are in non-standard places, you may want to use the `_ROOT_DIR` options +to specify their location. For example, to specify an OpenSSL location: + + $ cmake -DOPENSSL_ROOT_DIR=/tmp/openssl-3.3.2 .. + +Since these options are general to CMake, their +[documentation](https://cmake.org/documentation/) may be helpful. If +you have questions about dependencies, please [contact us](#getting-help). + +Running Tests +------------- + Once built, you can run the tests from the `build` directory with the command - $ make test + $ ctest -V Alternatively you can run the test suite directly using, - $ ./libgit2_clar + $ ./libgit2_tests + +Invoking the test suite directly is useful because it allows you to execute +individual tests, or groups of tests using the `-s` flag. For example, to +run the index tests: + + $ ./libgit2_tests -sindex + +To run a single test named `index::racy::diff`, which corresponds to +the test function +[`test_index_racy__diff`](https://github.com/libgit2/libgit2/blob/main/tests/libgit2/index/racy.c#L22): + + $ ./libgit2_tests -sindex::racy::diff + +The test suite will print a `.` for every passing test, and an `F` for any +failing test. An `S` indicates that a test was skipped because it is not +applicable to your platform or is particularly expensive. + +**Note:** There should be _no_ failing tests when you build an unmodified +source tree from a [release](https://github.com/libgit2/libgit2/releases), +or from the [main branch](https://github.com/libgit2/libgit2/tree/main). +Please contact us or +[open an issue](https://github.com/libgit2/libgit2/issues) +if you see test failures. + +Installation +------------ To install the library you can specify the install prefix by setting: $ cmake .. -DCMAKE_INSTALL_PREFIX=/install/prefix $ cmake --build . --target install -For more advanced use or questions about CMake please read . +Advanced Usage +-------------- + +For more advanced use or questions about CMake please read the +[CMake FAQ](https://cmake.org/Wiki/CMake_FAQ). The following CMake variables are declared: -- `BIN_INSTALL_DIR`: Where to install binaries to. -- `LIB_INSTALL_DIR`: Where to install libraries to. -- `INCLUDE_INSTALL_DIR`: Where to install headers to. +- `CMAKE_INSTALL_BINDIR`: Where to install binaries to. +- `CMAKE_INSTALL_LIBDIR`: Where to install libraries to. +- `CMAKE_INSTALL_INCLUDEDIR`: Where to install headers to. - `BUILD_SHARED_LIBS`: Build libgit2 as a Shared Library (defaults to ON) -- `BUILD_CLAR`: Build [Clar](https://github.com/vmg/clar)-based test suite (defaults to ON) -- `THREADSAFE`: Build libgit2 with threading support (defaults to ON) -- `STDCALL`: Build libgit2 as `stdcall`. Turn off for `cdecl` (Windows; defaults to ON) +- `BUILD_TESTS`: Build the unit and integration test suites (defaults to ON) +- `USE_THREADS`: Build libgit2 with threading support (defaults to ON) + +To list all build options and their current value, you can do the +following: + + # Create and set up a build directory + $ mkdir build && cd build + $ cmake .. + + # List all build options and their values + $ cmake -L Compiler and linker options --------------------------- -CMake lets you specify a few variables to control the behavior of the -compiler and linker. These flags are rarely used but can be useful for -64-bit to 32-bit cross-compilation. +There are several options that control the behavior of the compiler and +linker. These flags may be useful for cross-compilation or specialized +setups. - `CMAKE_C_FLAGS`: Set your own compiler flags +- `CMAKE_C_STANDARD`: the C standard to compile against; defaults to `C90` +- `CMAKE_C_EXTENSIONS`: whether compiler extensions are supported; defaults to `OFF` - `CMAKE_FIND_ROOT_PATH`: Override the search path for libraries - `ZLIB_LIBRARY`, `OPENSSL_SSL_LIBRARY` AND `OPENSSL_CRYPTO_LIBRARY`: Tell CMake where to find those specific libraries +- `LINK_WITH_STATIC_LIBRARIES`: Link only with static versions of +system libraries -MacOS X +macOS ------- -If you want to build a universal binary for Mac OS X, CMake sets it -all up for you if you use `-DCMAKE_OSX_ARCHITECTURES="i386;x86_64"` -when configuring. +If you'd like to work with Xcode, you can generate an Xcode project with "-G Xcode". -Windows + # Create and set up a build directory + $ mkdir build && cd build + $ cmake -G Xcode .. + +> [!TIP] +> Universal binary support: +> +> If you want to build a universal binary for macOS 11.0+, CMake sets it +> all up for you if you use `-DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"` +> when configuring. +> +> [Deprecated] If you want to build a universal binary for Mac OS X +> (10.4.4 ~ 10.6), CMake sets it all up for you if you use +> `-DCMAKE_OSX_ARCHITECTURES="i386;x86_64"` when configuring. + +iOS ------- -You need to run the CMake commands from the Visual Studio command -prompt, not the regular or Windows SDK one. Select the right generator -for your version with the `-G "Visual Studio X" option. +1. Get an iOS cmake toolchain File: -See [the website](http://libgit2.github.com/docs/guides/build-and-link/) -for more detailed instructions. +You can use a pre-existing toolchain file like [ios-cmake](https://github.com/leetal/ios-cmake) or write your own. + +2. Specify the toolchain and system Name: + +- The CMAKE_TOOLCHAIN_FILE variable points to the toolchain file for iOS. +- The CMAKE_SYSTEM_NAME should be set to iOS. + +3. Example Command: + +Assuming you're using the ios-cmake toolchain, the command might look like this: + +``` +cmake -G Xcode -DCMAKE_TOOLCHAIN_FILE=path/to/ios.toolchain.cmake -DCMAKE_SYSTEM_NAME=iOS -DPLATFORM=OS64 .. +``` + +4. Build the Project: + +After generating the project, open the .xcodeproj file in Xcode, select your iOS device or simulator as the target, and build your project. Android ------- @@ -210,6 +463,23 @@ with full path to the toolchain): Add `-DCMAKE_TOOLCHAIN_FILE={pathToToolchainFile}` to cmake command when configuring. +MinGW +----- + +If you want to build the library in MinGW environment with SSH support +enabled, you may need to pass +`-DCMAKE_LIBRARY_PATH="${MINGW_PREFIX}/${MINGW_CHOST}/lib/"` flag +to CMake when configuring. This is because CMake cannot find the +Win32 libraries in MinGW folders by default and you might see an +error message stating that CMake could not resolve `ws2_32` library +during configuration. + +Another option would be to install `msys2-w32api-runtime` package before +configuring. This package installs the Win32 libraries into `/usr/lib` +folder which is by default recognized as the library path by CMake. +Please note though that this package is meant for MSYS subsystem which +is different from MinGW. + Language Bindings ================================== @@ -223,18 +493,24 @@ Here are the bindings to libgit2 that are currently available: * dlibgit * Delphi * GitForDelphi + * libgit2-delphi * Erlang * Geef * Go * git2go * GObject * libgit2-glib +* Guile + * Guile-Git * Haskell * hgit2 * Java * Jagged + * Git24j +* Javascript / WebAssembly ( browser and nodejs ) + * WASM-git * Julia - * LibGit2.jl + * LibGit2.jl * Lua * luagit2 * .NET @@ -249,20 +525,25 @@ Here are the bindings to libgit2 that are currently available: * parrot-libgit2 * Perl * Git-Raw +* Pharo Smalltalk + * libgit2-pharo-bindings * PHP - * php-git -* PowerShell - * PSGit + * php-git2 * Python * pygit2 * R + * gert * git2r * Ruby * Rugged * Rust - * git2-rs + * git2-rs * Swift * SwiftGit2 + * SwiftGitX + * swift-libgit2 +* Tcl + * lg2 * Vala * libgit2.vapi @@ -277,17 +558,18 @@ We welcome new contributors! We have a number of issues marked as and ["easy fix"](https://github.com/libgit2/libgit2/issues?utf8=✓&q=is%3Aissue+is%3Aopen+label%3A%22easy+fix%22) that are good places to jump in and get started. There's much more detailed -information in our list of [outstanding projects](PROJECTS.md). +information in our list of [outstanding projects](docs/projects.md). -Please be sure to check the [contribution guidelines](CONTRIBUTING.md) to -understand our workflow, and the libgit2 [coding conventions](CONVENTIONS.md). +Please be sure to check the [contribution guidelines](docs/contributing.md) +to understand our workflow, and the libgit2 +[coding conventions](docs/conventions.md). License ================================== -`libgit2` is under GPL2 **with linking exception**. This means you can link to -and use the library from any program, proprietary or open source; paid or -gratis. However, if you modify libgit2 itself, you must distribute the -source to your modified version of libgit2. +`libgit2` is under GPL2 **with linking exception**. This means you can +link to and use the library from any program, proprietary or open source; +paid or gratis. However, if you modify libgit2 itself, you must distribute +the source to your modified version of libgit2. See the [COPYING file](COPYING) for the full license text. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..914e660b26d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security Policy + +## Supported Versions + +This project will always provide security fixes for the latest two released +versions. E.g. if the latest version is v0.28.x, then we will provide security +fixes for both v0.28.x and v0.27.y, but no earlier versions. + +## Reporting a Vulnerability + +In case you think to have found a security issue with libgit2, please do not +open a public issue. Instead, you can report the issue to the private mailing +list [security@libgit2.com](mailto:security@libgit2.com). We will acknowledge +receipt of your message in at most three days and try to clarify further steps. diff --git a/THREADING.md b/THREADING.md deleted file mode 100644 index 430bca858d3..00000000000 --- a/THREADING.md +++ /dev/null @@ -1,115 +0,0 @@ -Threads in libgit2 -================== - -You may safely use any libgit2 object from any thread, though there -may be issues depending on the cryptographic libraries libgit2 or its -dependencies link to (more on this later). For libgit2 itself, -provided you take the following into consideration you won't run into -issues: - -Sharing objects ---------------- - -Use an object from a single thread at a time. Most data structures do -not guard against concurrent access themselves. This is because they -are rarely used in isolation and it makes more sense to synchronize -access via a larger lock or similar mechanism. - -There are some objects which are read-only/immutable and are thus safe -to share across threads, such as references and configuration -snapshots. - -Error messages --------------- - -The error message is thread-local. The `giterr_last()` call must -happen on the same thread as the error in order to get the -message. Often this will be the case regardless, but if you use -something like the [GCD](http://en.wikipedia.org/wiki/Grand_Central_Dispatch) -on Mac OS X (where code is executed on an arbitrary thread), the code -must make sure to retrieve the error code on the thread where the error -happened. - -Threads and cryptographic libraries -======================================= - -On Windows ----------- - -When built as a native Windows DLL, libgit2 uses WinCNG and WinHTTP, -both of which are thread-safe. You do not need to do anything special. - -When using libssh2 which itself uses WinCNG, there are no special -steps necessary. If you are using a MinGW or similar environment where -libssh2 uses OpenSSL or libgcrypt, then the general case affects -you. - -On Mac OS X ------------ - -By default we use libcurl to perform the encryption. The -system-provided libcurl uses SecureTransport, so no special steps are -necessary. If you link against another libcurl (e.g. from homebrew) -refer to the general case. - -If the option to use libcurl was deactivated, the library makes use of -CommonCrypto and SecureTransport for cryptographic support. These are -thread-safe and you do not need to do anything special. - -Note that libssh2 may still use OpenSSL itself. In that case, the -general case still affects you if you use ssh. - -General Case ------------- - -If it's available, by default we use libcurl to provide HTTP tunneling support, -which may be linked against a number of cryptographic libraries and has its -own -[recommendations for thread safety](https://curl.haxx.se/libcurl/c/threadsafe.html). - -If there are no alternative TLS implementations (currently only -SecureTransport), libgit2 uses OpenSSL in order to use HTTPS as a transport. -OpenSSL is thread-safe starting at version 1.1.0. If your copy of libgit2 is -linked against that version, you do not need to take any further steps. - -Older versions of OpenSSL are made to be thread-implementation agnostic, and the -users of the library must set which locking function it should use. libgit2 -cannot know what to set as the user of libgit2 may also be using OpenSSL independently and -the locking settings must then live outside the lifetime of libgit2. - -Even if libgit2 doesn't use OpenSSL directly, OpenSSL can still be used by -libssh2 or libcurl depending on the configuration. If OpenSSL is used by -more than one library, you only need to set up threading for OpenSSL once. - -If libgit2 is linked against OpenSSL, it provides a last-resort convenience function -`git_openssl_set_locking()` (available in `sys/openssl.h`) to use the -platform-native mutex mechanisms to perform the locking, which you can use -if you do not want to use OpenSSL outside of libgit2, or you -know that libgit2 will outlive the rest of the operations. It is then not -safe to use OpenSSL multi-threaded after libgit2's shutdown function -has been called. Note `git_openssl_set_locking()` only works if -libgit2 uses OpenSSL directly - if OpenSSL is only used as a dependency -of libssh2 or libcurl as described above, `git_openssl_set_locking()` is a no-op. - -If your programming language offers a package/bindings for OpenSSL, -you should very strongly prefer to use that in order to set up -locking, as they provide a level of coordination which is impossible -when using this function. - -See the -[OpenSSL documentation](https://www.openssl.org/docs/crypto/threads.html) -on threading for more details, and http://trac.libssh2.org/wiki/MultiThreading -for a specific example of providing the threading callbacks. - -libssh2 may be linked against OpenSSL or libgcrypt. If it uses OpenSSL, -see the above paragraphs. If it uses libgcrypt, then you need to -set up its locking before using it multi-threaded. libgit2 has no -direct connection to libgcrypt and thus has no convenience functions for -it (but libgcrypt has macros). Read libgcrypt's -[threading documentation for more information](http://www.gnupg.org/documentation/manuals/gcrypt/Multi_002dThreading.html) - -It is your responsibility as an application author or packager to know -what your dependencies are linked against and to take the appropriate -steps to ensure the cryptographic libraries are thread-safe. We agree -that this situation is far from ideal but at this time it is something -the application authors need to deal with. diff --git a/api.docurium b/api.docurium index 9e17817dbec..bf733273b8c 100644 --- a/api.docurium +++ b/api.docurium @@ -1,7 +1,7 @@ { "name": "libgit2", "github": "libgit2/libgit2", - "input": "include/git2", + "input": "include", "prefix": "git_", "output": "docs", "branch": "gh-pages", diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index fb3fff7ddcc..00000000000 --- a/appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: '{build}' -branches: - only: - - master - - appveyor - - /^maint.*/ -environment: - GITTEST_INVASIVE_FS_STRUCTURE: 1 - GITTEST_INVASIVE_FS_SIZE: 1 - - matrix: - - GENERATOR: "Visual Studio 11" - ARCH: 32 - - GENERATOR: "Visual Studio 11 Win64" - ARCH: 64 - - GENERATOR: "MSYS Makefiles" - ARCH: i686 # this is for 32-bit MinGW-w64 - - GENERATOR: "MSYS Makefiles" - ARCH: 64 -cache: -- i686-4.9.2-release-win32-sjlj-rt_v3-rev1.7z -- x86_64-4.9.2-release-win32-seh-rt_v3-rev1.7z - -build_script: -- ps: | - mkdir build - cd build - if ($env:GENERATOR -ne "MSYS Makefiles") { - cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON -D MSVC_CRTDBG=ON .. -G"$env:GENERATOR" - cmake --build . --config Debug - } -- cmd: | - if "%GENERATOR%"=="MSYS Makefiles" (C:\MinGW\msys\1.0\bin\sh --login /c/projects/libgit2/script/appveyor-mingw.sh) -test_script: -- ps: | - $ErrorActionPreference="Stop" - Start-FileDownload https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar -FileName poxyproxy.jar - # Run this early so we know it's ready by the time we need it - $proxyJob = Start-Job { java -jar $Env:APPVEYOR_BUILD_FOLDER\build\poxyproxy.jar -d --port 8080 --credentials foo:bar } - ctest -V -R libgit2_clar - $env:GITTEST_REMOTE_URL="https://github.com/libgit2/non-existent" - $env:GITTEST_REMOTE_USER="libgit2test" - ctest -V -R libgit2_clar-cred_callback - Receive-Job -Job $proxyJob - $env:GITTEST_REMOTE_PROXY_URL = "http://foo:bar@localhost:8080" - ctest -V -R libgit2_clar-proxy_credentials_in_url - $env:GITTEST_REMOTE_PROXY_URL = "http://localhost:8080" - $env:GITTEST_REMOTE_PROXY_USER = "foo" - $env:GITTEST_REMOTE_PROXY_PASS = "bar" - ctest -V -R libgit2_clar-proxy_credentials_request diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..1b31354e50a --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(libgit2) diff --git a/benchmarks/cli/README.md b/benchmarks/cli/README.md new file mode 100644 index 00000000000..f66b27aea3b --- /dev/null +++ b/benchmarks/cli/README.md @@ -0,0 +1,121 @@ +# libgit2 benchmarks + +This folder contains the individual benchmark tests for libgit2, +meant for understanding the performance characteristics of libgit2, +comparing your development code to the existing libgit2 code, or +comparing libgit2 to the git reference implementation. + +## Running benchmark tests + +Benchmark tests can be run in several different ways: running all +benchmarks, running one (or more) suite of benchmarks, or running a +single individual benchmark. You can target either an individual +version of a CLI, or you can A/B test a baseline CLI against a test +CLI. + +### Specifying the command-line interface to test + +By default, the `git` in your path is benchmarked. Use the +`-c` (or `--cli`) option to specify the command-line interface +to test. + +Example: `libgit2_bench --cli git2_cli` will run the tests against +`git2_cli`. + +### Running tests to compare two different implementations + +You can compare a baseline command-line interface against a test +command-line interface using the `-b (or `--baseline-cli`) option. + +Example: `libgit2_bench --baseline-cli git --cli git2_cli` will +run the tests against both `git` and `git2_cli`. + +### Running individual benchmark tests + +Similar to how a test suite or individual test is specified in +[clar](https://github.com/clar-test/clar), the `-s` (or `--suite`) +option may be used to specify the suite or individual test to run. +Like clar, the suite and test name are separated by `::`, and like +clar, this is a prefix match. + +Examples: +* `libgit2_bench -shash_object` will run the tests in the + `hash_object` suite. +* `libgit2_bench -shash_object::random_1kb` will run the + `hash_object::random_1kb` test. +* `libgit2_bench -shash_object::random` will run all the tests that + begin with `hash_object::random`. + +## Writing benchmark tests + +Benchmark tests are meant to be easy to write. Each individual +benchmark is a shell script that allows it to do set up (eg, creating +or cloning a repository, creating temporary files, etc), then running +benchmarks, then teardown. + +The `benchmark_helpers.sh` script provides many helpful utility +functions to allow for cross-platform benchmarking, as well as a +wrapper for `hyperfine` that is suited to testing libgit2. +Note that the helper script must be included first, at the beginning +of the benchmark test. + +### Benchmark example + +This simplistic example compares the speed of running the `git help` +command in the baseline CLI to the test CLI. + +```bash +#!/bin/bash -e + +# include the benchmark library +. "$(dirname "$0")/benchmark_helpers.sh" + +# run the "help" command; this will benchmark `git2_cli help` +gitbench help +``` + +### Naming + +The filename of the benchmark itself is important. A benchmark's +filename should be the name of the benchmark suite, followed by two +underscores, followed by the name of the benchmark. For example, +`hash-object__random_1kb` is the `random_1kb` test in the `hash-object` +suite. + +### Options + +The `gitbench` function accepts several options. + +* `--sandbox ` + The name of a test resource (in the `tests/resources` directory). + This will be copied as-is to the sandbox location before test + execution. This is copied _before_ the `prepare` script is run. + This option may be specified multiple times. +* `--repository ` + The name of a test resource repository (in the `tests/resources` + directory). This repository will be copied into a sandbox location + before test execution, and your test will run in this directory. + This is copied _before_ the `prepare` script is run. +* `--prepare + +Flame Graph + +Reset Zoom +Search + +unix`mutex_enter (195 samples, 0.34%) + + + +genunix`as_fault (12 samples, 0.02%) + + + +genunix`disp_lock_exit (27 samples, 0.05%) + + + +genunix`vsd_free (17 samples, 0.03%) + + + +genunix`pn_fixslash (44 samples, 0.08%) + + + +unix`mutex_exit (105 samples, 0.18%) + + + +genunix`falloc (1,363 samples, 2.37%) +g.. + + +genunix`traverse (30 samples, 0.05%) + + + +genunix`fop_lookup (55 samples, 0.10%) + + + +genunix`kmem_cache_free (29 samples, 0.05%) + + + +lofs`makelonode (39 samples, 0.07%) + + + +genunix`vsd_free (155 samples, 0.27%) + + + +unix`strlen (2,659 samples, 4.63%) +unix`.. + + +unix`clear_int_flag (180 samples, 0.31%) + + + +unix`mutex_exit (38 samples, 0.07%) + + + +genunix`kmem_cpu_reload (5 samples, 0.01%) + + + +unix`mutex_exit (26 samples, 0.05%) + + + +genunix`vn_vfslocks_getlock (47 samples, 0.08%) + + + +unix`bzero (8 samples, 0.01%) + + + +genunix`vn_exists (50 samples, 0.09%) + + + +unix`mutex_enter (727 samples, 1.27%) + + + +genunix`kmem_cache_alloc (179 samples, 0.31%) + + + +unix`mutex_enter (905 samples, 1.58%) + + + +genunix`ufalloc (10 samples, 0.02%) + + + +genunix`vn_rele (25 samples, 0.04%) + + + +genunix`vn_exists (17 samples, 0.03%) + + + +unix`lock_try (778 samples, 1.35%) + + + +genunix`rwst_enter_common (314 samples, 0.55%) + + + +genunix`fsop_root (62 samples, 0.11%) + + + +lofs`table_lock_enter (44 samples, 0.08%) + + + +unix`mutex_exit (138 samples, 0.24%) + + + +unix`mutex_enter (316 samples, 0.55%) + + + +genunix`kmem_cache_free (5 samples, 0.01%) + + + +unix`preempt (14 samples, 0.02%) + + + +genunix`vn_alloc (1,189 samples, 2.07%) +g.. + + +genunix`kmem_cache_alloc (126 samples, 0.22%) + + + +genunix`vfs_getops (157 samples, 0.27%) + + + +lofs`lsave (27 samples, 0.05%) + + + +unix`tsc_read (160 samples, 0.28%) + + + +lofs`lfind (26 samples, 0.05%) + + + +unix`atomic_add_64 (205 samples, 0.36%) + + + +unix`mutex_enter (320 samples, 0.56%) + + + +genunix`traverse (17 samples, 0.03%) + + + +unix`mutex_enter (197 samples, 0.34%) + + + +genunix`vn_mountedvfs (20 samples, 0.03%) + + + +genunix`audit_unfalloc (340 samples, 0.59%) + + + +genunix`kmem_cache_free (209 samples, 0.36%) + + + +genunix`kmem_zalloc (13 samples, 0.02%) + + + +genunix`thread_lock (33 samples, 0.06%) + + + +unix`tsc_read (186 samples, 0.32%) + + + +genunix`vn_vfsrlock (12 samples, 0.02%) + + + +lofs`lo_inactive (21 samples, 0.04%) + + + +genunix`rwst_destroy (20 samples, 0.03%) + + + +unix`mutex_enter (379 samples, 0.66%) + + + +genunix`vn_setops (41 samples, 0.07%) + + + +genunix`vn_recycle (33 samples, 0.06%) + + + +lofs`lo_inactive (6,307 samples, 10.98%) +lofs`lo_inactive + + +lofs`table_lock_enter (220 samples, 0.38%) + + + +genunix`cv_broadcast (25 samples, 0.04%) + + + +unix`mutex_exit (358 samples, 0.62%) + + + +genunix`kmem_cache_alloc (234 samples, 0.41%) + + + +unix`rw_enter (525 samples, 0.91%) + + + +unix`membar_consumer (237 samples, 0.41%) + + + +unix`swtch (5 samples, 0.01%) + + + +genunix`rwst_enter_common (32 samples, 0.06%) + + + +lofs`freelonode (5,313 samples, 9.25%) +lofs`freelonode + + +genunix`vn_openat (46,342 samples, 80.68%) +genunix`vn_openat + + +genunix`vn_rele (19 samples, 0.03%) + + + +genunix`proc_exit (5 samples, 0.01%) + + + +unix`mutex_exit (512 samples, 0.89%) + + + +genunix`kmem_free (35 samples, 0.06%) + + + +unix`mutex_enter (252 samples, 0.44%) + + + +genunix`rwst_exit (12 samples, 0.02%) + + + +genunix`crgetuid (22 samples, 0.04%) + + + +genunix`kmem_free (17 samples, 0.03%) + + + +unix`mutex_init (53 samples, 0.09%) + + + +ufs`ufs_iaccess (648 samples, 1.13%) + + + +all (57,441 samples, 100%) + + + +genunix`fop_inactive (6,689 samples, 11.64%) +genunix`fop_inact.. + + +genunix`kmem_cache_alloc (9 samples, 0.02%) + + + +genunix`kmem_cache_free (184 samples, 0.32%) + + + +genunix`pn_get_buf (13 samples, 0.02%) + + + +unix`strlen (107 samples, 0.19%) + + + +unix`mutex_exit (46 samples, 0.08%) + + + +genunix`post_syscall (12 samples, 0.02%) + + + +unix`mutex_init (38 samples, 0.07%) + + + +unix`rw_exit (439 samples, 0.76%) + + + +lofs`lo_lookup (65 samples, 0.11%) + + + +genunix`clear_stale_fd (44 samples, 0.08%) + + + +unix`mutex_enter (238 samples, 0.41%) + + + +genunix`pn_get_buf (687 samples, 1.20%) + + + +genunix`vn_free (1,663 samples, 2.90%) +ge.. + + +unix`mutex_enter (980 samples, 1.71%) + + + +genunix`crhold (5 samples, 0.01%) + + + +unix`mutex_exit (59 samples, 0.10%) + + + +genunix`vn_reinit (48 samples, 0.08%) + + + +genunix`vfs_getops (21 samples, 0.04%) + + + +genunix`open (49,669 samples, 86.47%) +genunix`open + + +genunix`kmem_cache_alloc (39 samples, 0.07%) + + + +genunix`vn_vfslocks_getlock (79 samples, 0.14%) + + + +unix`clear_int_flag (39 samples, 0.07%) + + + +genunix`kmem_cache_free (215 samples, 0.37%) + + + +unix`mutex_destroy (53 samples, 0.09%) + + + +genunix`vn_vfsunlock (3,578 samples, 6.23%) +genunix`.. + + +genunix`dnlc_lookup (1,843 samples, 3.21%) +gen.. + + +genunix`lookupnameatcred (45,978 samples, 80.04%) +genunix`lookupnameatcred + + +genunix`crgetmapped (41 samples, 0.07%) + + + +genunix`anon_zero (7 samples, 0.01%) + + + +genunix`rwst_tryenter (628 samples, 1.09%) + + + +unix`mutex_enter (309 samples, 0.54%) + + + +genunix`vn_rele (14 samples, 0.02%) + + + +genunix`vn_setpath (1,969 samples, 3.43%) +gen.. + + +unix`mutex_enter (111 samples, 0.19%) + + + +genunix`cv_broadcast (40 samples, 0.07%) + + + +genunix`kmem_cache_alloc (66 samples, 0.11%) + + + +genunix`audit_getstate (21 samples, 0.04%) + + + +genunix`vn_setpath (58 samples, 0.10%) + + + +genunix`open (17 samples, 0.03%) + + + +unix`bcopy (896 samples, 1.56%) + + + +unix`mutex_enter (99 samples, 0.17%) + + + +genunix`traverse (5,557 samples, 9.67%) +genunix`traverse + + +genunix`pn_getcomponent (41 samples, 0.07%) + + + +unix`mutex_enter (640 samples, 1.11%) + + + +unix`mutex_destroy (176 samples, 0.31%) + + + +unix`lwp_getdatamodel (6 samples, 0.01%) + + + +genunix`unfalloc (39 samples, 0.07%) + + + +genunix`syscall_mstate (355 samples, 0.62%) + + + +genunix`cv_init (65 samples, 0.11%) + + + +unix`mutex_enter (95 samples, 0.17%) + + + +unix`bcmp (42 samples, 0.07%) + + + +unix`mutex_exit (350 samples, 0.61%) + + + +genunix`kmem_free (288 samples, 0.50%) + + + +unix`mutex_exit (58 samples, 0.10%) + + + +genunix`kmem_alloc (32 samples, 0.06%) + + + +unix`mutex_exit (356 samples, 0.62%) + + + +unix`mutex_init (46 samples, 0.08%) + + + +genunix`rwst_init (173 samples, 0.30%) + + + +genunix`rwst_enter_common (28 samples, 0.05%) + + + +genunix`openat (49,647 samples, 86.43%) +genunix`openat + + +unix`mutex_enter (303 samples, 0.53%) + + + +lofs`lfind (278 samples, 0.48%) + + + +unix`mutex_exit (90 samples, 0.16%) + + + +genunix`cv_init (49 samples, 0.09%) + + + +unix`tsc_gethrtimeunscaled (43 samples, 0.07%) + + + +genunix`rwst_tryenter (32 samples, 0.06%) + + + +genunix`pn_fixslash (14 samples, 0.02%) + + + +genunix`gethrtime_unscaled (420 samples, 0.73%) + + + +genunix`post_syscall (4,245 samples, 7.39%) +genunix`po.. + + +genunix`kmem_zalloc (280 samples, 0.49%) + + + +genunix`vn_alloc (20 samples, 0.03%) + + + +genunix`vn_mountedvfs (43 samples, 0.07%) + + + +genunix`audit_getstate (15 samples, 0.03%) + + + +zfs`zfs_lookup (22 samples, 0.04%) + + + +genunix`crgetuid (6 samples, 0.01%) + + + +unix`copystr (598 samples, 1.04%) + + + +unix`i_ddi_splhigh (23 samples, 0.04%) + + + +unix`trap (13 samples, 0.02%) + + + +genunix`audit_getstate (27 samples, 0.05%) + + + +genunix`vn_mountedvfs (56 samples, 0.10%) + + + +unix`mutex_destroy (17 samples, 0.03%) + + + +genunix`cv_broadcast (14 samples, 0.02%) + + + +genunix`segvn_fault (11 samples, 0.02%) + + + +genunix`vn_rele (39 samples, 0.07%) + + + +genunix`kmem_free (457 samples, 0.80%) + + + +genunix`vn_vfsunlock (20 samples, 0.03%) + + + +genunix`vn_vfslocks_rele (34 samples, 0.06%) + + + +unix`atomic_cas_64 (318 samples, 0.55%) + + + +unix`mutex_enter (337 samples, 0.59%) + + + +unix`do_splx (31 samples, 0.05%) + + + +genunix`ufalloc_file (20 samples, 0.03%) + + + +genunix`fd_reserve (35 samples, 0.06%) + + + +genunix`copen (49,444 samples, 86.08%) +genunix`copen + + +unix`mutex_enter (279 samples, 0.49%) + + + +unix`0xfffffffffb800c91 (4,361 samples, 7.59%) +unix`0xfff.. + + +genunix`crgetmapped (55 samples, 0.10%) + + + +genunix`cv_init (56 samples, 0.10%) + + + +genunix`dnlc_lookup (26 samples, 0.05%) + + + +genunix`kmem_alloc (11 samples, 0.02%) + + + +genunix`cv_init (53 samples, 0.09%) + + + +unix`copyinstr (25 samples, 0.04%) + + + +genunix`gethrtime_unscaled (203 samples, 0.35%) + + + +genunix`kmem_cache_alloc (11 samples, 0.02%) + + + +genunix`vn_free (26 samples, 0.05%) + + + +unix`mutex_exit (149 samples, 0.26%) + + + +genunix`vn_recycle (319 samples, 0.56%) + + + +genunix`vn_rele (64 samples, 0.11%) + + + +unix`bcmp (11 samples, 0.02%) + + + +genunix`kmem_cache_free (154 samples, 0.27%) + + + +unix`lock_clear_splx (28 samples, 0.05%) + + + +genunix`unfalloc (729 samples, 1.27%) + + + +genunix`fop_lookup (85 samples, 0.15%) + + + +zfs`specvp_check (10 samples, 0.02%) + + + +genunix`lookupnameatcred (22 samples, 0.04%) + + + +unix`tsc_read (367 samples, 0.64%) + + + +genunix`memcmp (38 samples, 0.07%) + + + +unix`splx (6 samples, 0.01%) + + + +unix`mutex_exit (95 samples, 0.17%) + + + +genunix`gethrtime_unscaled (7 samples, 0.01%) + + + +genunix`rwst_init (13 samples, 0.02%) + + + +genunix`audit_getstate (31 samples, 0.05%) + + + +genunix`kmem_cache_alloc (32 samples, 0.06%) + + + +genunix`disp_lock_exit (2,096 samples, 3.65%) +genu.. + + +unix`mutex_exit (49 samples, 0.09%) + + + +unix`copyinstr (18 samples, 0.03%) + + + +ufs`ufs_lookup (46 samples, 0.08%) + + + +genunix`clear_stale_fd (10 samples, 0.02%) + + + +genunix`rwst_destroy (296 samples, 0.52%) + + + +genunix`syscall_mstate (1,336 samples, 2.33%) +g.. + + +genunix`kmem_alloc (934 samples, 1.63%) + + + +unix`atomic_add_32 (325 samples, 0.57%) + + + +unix`mutex_enter (947 samples, 1.65%) + + + +unix`mutex_exit (56 samples, 0.10%) + + + +unix`mutex_enter (318 samples, 0.55%) + + + +lofs`lo_root (80 samples, 0.14%) + + + +genunix`lookuppnvp (44,242 samples, 77.02%) +genunix`lookuppnvp + + +genunix`lookupnameat (46,075 samples, 80.21%) +genunix`lookupnameat + + +unix`setbackdq (5 samples, 0.01%) + + + +lofs`lo_root (31 samples, 0.05%) + + + +genunix`kmem_cache_alloc (17 samples, 0.03%) + + + +unix`mutex_exit (212 samples, 0.37%) + + + +genunix`vn_vfsrlock (2,414 samples, 4.20%) +genun.. + + +genunix`vfs_matchops (28 samples, 0.05%) + + + +unix`prunstop (36 samples, 0.06%) + + + +unix`mutex_exit (155 samples, 0.27%) + + + +unix`mutex_init (31 samples, 0.05%) + + + +unix`atomic_add_32_nv (100 samples, 0.17%) + + + +genunix`lookupnameat (69 samples, 0.12%) + + + +unix`_sys_rtt (6 samples, 0.01%) + + + +genunix`kmem_cache_alloc (49 samples, 0.09%) + + + +unix`tsc_gethrtimeunscaled (17 samples, 0.03%) + + + +genunix`fop_lookup (29,216 samples, 50.86%) +genunix`fop_lookup + + +unix`mutex_exit (142 samples, 0.25%) + + + +genunix`crgetmapped (31 samples, 0.05%) + + + +unix`do_splx (1,993 samples, 3.47%) +uni.. + + +genunix`kmem_cache_free (22 samples, 0.04%) + + + +unix`mutex_enter (95 samples, 0.17%) + + + +genunix`crhold (11 samples, 0.02%) + + + +unix`mutex_enter (823 samples, 1.43%) + + + +unix`mutex_exit (29 samples, 0.05%) + + + +genunix`vn_vfsrlock (3,342 samples, 5.82%) +genunix.. + + +unix`tsc_gethrtimeunscaled (13 samples, 0.02%) + + + +genunix`vn_rele (73 samples, 0.13%) + + + +unix`mutex_exit (337 samples, 0.59%) + + + +genunix`vn_vfslocks_getlock (973 samples, 1.69%) + + + +zfs`specvp_check (20 samples, 0.03%) + + + +genunix`vsd_free (14 samples, 0.02%) + + + +unix`mutex_enter (314 samples, 0.55%) + + + +genunix`cv_destroy (81 samples, 0.14%) + + + +genunix`cv_broadcast (25 samples, 0.04%) + + + +unix`mutex_enter (122 samples, 0.21%) + + + +unix`mutex_exit (55 samples, 0.10%) + + + +genunix`set_errno (24 samples, 0.04%) + + + +genunix`cv_destroy (42 samples, 0.07%) + + + +genunix`fd_find (13 samples, 0.02%) + + + +genunix`vn_invalid (47 samples, 0.08%) + + + +genunix`vfs_matchops (336 samples, 0.58%) + + + +unix`tsc_gethrtimeunscaled (59 samples, 0.10%) + + + +genunix`fop_inactive (39 samples, 0.07%) + + + +genunix`kmem_free (693 samples, 1.21%) + + + +genunix`syscall_mstate (412 samples, 0.72%) + + + +genunix`thread_lock (670 samples, 1.17%) + + + +lofs`lsave (162 samples, 0.28%) + + + +unix`atomic_add_64 (95 samples, 0.17%) + + + +genunix`audit_getstate (66 samples, 0.11%) + + + +genunix`dnlc_lookup (70 samples, 0.12%) + + + +genunix`vn_mountedvfs (30 samples, 0.05%) + + + +genunix`cv_broadcast (19 samples, 0.03%) + + + +genunix`kmem_alloc (533 samples, 0.93%) + + + +unix`mutex_exit (160 samples, 0.28%) + + + +genunix`memcmp (38 samples, 0.07%) + + + +unix`strlen (1,238 samples, 2.16%) +u.. + + +genunix`lookuppnatcred (12 samples, 0.02%) + + + +genunix`crfree (13 samples, 0.02%) + + + +lofs`table_lock_enter (43 samples, 0.07%) + + + +genunix`rwst_exit (18 samples, 0.03%) + + + +genunix`cv_destroy (31 samples, 0.05%) + + + +genunix`rwst_init (236 samples, 0.41%) + + + +genunix`vn_vfslocks_rele (1,420 samples, 2.47%) +ge.. + + +genunix`falloc (36 samples, 0.06%) + + + +genunix`setf (187 samples, 0.33%) + + + +zfs`zfs_fastaccesschk_execute (50 samples, 0.09%) + + + +genunix`vn_vfslocks_getlock (120 samples, 0.21%) + + + +genunix`fd_reserve (9 samples, 0.02%) + + + +genunix`vn_setops (160 samples, 0.28%) + + + +unix`sys_syscall (51,908 samples, 90.37%) +unix`sys_syscall + + +genunix`kmem_free (115 samples, 0.20%) + + + +genunix`vsd_free (48 samples, 0.08%) + + + +genunix`rexit (5 samples, 0.01%) + + + +genunix`vn_mountedvfs (11 samples, 0.02%) + + + +genunix`lookuppnatcred (44,681 samples, 77.79%) +genunix`lookuppnatcred + + +unix`splr (92 samples, 0.16%) + + + +genunix`vn_vfsrlock (13 samples, 0.02%) + + + +unix`mutex_exit (371 samples, 0.65%) + + + +genunix`kmem_cache_free (5 samples, 0.01%) + + + +genunix`dnlc_lookup (263 samples, 0.46%) + + + +genunix`audit_unfalloc (32 samples, 0.06%) + + + +unix`0xfffffffffb8001d6 (13 samples, 0.02%) + + + +genunix`rwst_destroy (146 samples, 0.25%) + + + +genunix`gethrtime_unscaled (182 samples, 0.32%) + + + +unix`mutex_enter (575 samples, 1.00%) + + + +unix`mutex_exit (148 samples, 0.26%) + + + +genunix`ufalloc_file (294 samples, 0.51%) + + + +unix`mutex_exit (163 samples, 0.28%) + + + +unix`membar_consumer (106 samples, 0.18%) + + + +genunix`crgetmapped (36 samples, 0.06%) + + + +genunix`memcmp (277 samples, 0.48%) + + + +genunix`cv_destroy (77 samples, 0.13%) + + + +genunix`kmem_cache_free (116 samples, 0.20%) + + + +genunix`kmem_cache_alloc (29 samples, 0.05%) + + + +genunix`fd_reserve (8 samples, 0.01%) + + + +zfs`zfs_lookup (946 samples, 1.65%) + + + +genunix`kmem_alloc (795 samples, 1.38%) + + + +unix`tsc_gethrtimeunscaled (11 samples, 0.02%) + + + +genunix`segvn_faultpage (7 samples, 0.01%) + + + +genunix`set_errno (9 samples, 0.02%) + + + +unix`splr (400 samples, 0.70%) + + + +genunix`rwst_destroy (32 samples, 0.06%) + + + +genunix`rwst_init (28 samples, 0.05%) + + + +unix`atomic_add_32 (292 samples, 0.51%) + + + +unix`0xfffffffffb800ca0 (517 samples, 0.90%) + + + +genunix`syscall_mstate (89 samples, 0.15%) + + + +genunix`kmem_alloc (73 samples, 0.13%) + + + +genunix`vn_vfsunlock (40 samples, 0.07%) + + + +unix`mutex_enter (1,202 samples, 2.09%) +u.. + + +lofs`makelfsnode (28 samples, 0.05%) + + + +unix`0xfffffffffb800c86 (472 samples, 0.82%) + + + +genunix`vn_rele (6,943 samples, 12.09%) +genunix`vn_rele + + +unix`mutex_exit (56 samples, 0.10%) + + + +genunix`kmem_cache_free (51 samples, 0.09%) + + + +genunix`gethrtime_unscaled (11 samples, 0.02%) + + + +unix`pagefault (13 samples, 0.02%) + + + +genunix`secpolicy_vnode_access2 (217 samples, 0.38%) + + + +genunix`vn_vfslocks_getlock (1,357 samples, 2.36%) +g.. + + +unix`bcmp (295 samples, 0.51%) + + + +unix`mutex_enter (97 samples, 0.17%) + + + +unix`membar_consumer (123 samples, 0.21%) + + + +genunix`audit_getstate (16 samples, 0.03%) + + + +unix`mutex_enter (455 samples, 0.79%) + + + +lofs`makelonode (4,212 samples, 7.33%) +lofs`makel.. + + +genunix`kmem_cache_alloc (168 samples, 0.29%) + + + +genunix`vn_vfslocks_getlock (62 samples, 0.11%) + + + +genunix`secpolicy_vnode_access2 (72 samples, 0.13%) + + + +genunix`kmem_cache_free (73 samples, 0.13%) + + + +genunix`vn_reinit (424 samples, 0.74%) + + + +genunix`pn_getcomponent (454 samples, 0.79%) + + + +genunix`fsop_root (297 samples, 0.52%) + + + +genunix`crgetuid (30 samples, 0.05%) + + + +genunix`kmem_free (785 samples, 1.37%) + + + +unix`mutex_exit (171 samples, 0.30%) + + + +genunix`crgetmapped (58 samples, 0.10%) + + + +unix`mutex_enter (299 samples, 0.52%) + + + +genunix`rwst_exit (167 samples, 0.29%) + + + +genunix`audit_falloc (8 samples, 0.01%) + + + +genunix`rwst_exit (110 samples, 0.19%) + + + +genunix`exit (5 samples, 0.01%) + + + +unix`mutex_exit (250 samples, 0.44%) + + + +lofs`freelonode (35 samples, 0.06%) + + + +genunix`rwst_tryenter (37 samples, 0.06%) + + + +ufs`ufs_iaccess (91 samples, 0.16%) + + + +unix`tsc_gethrtimeunscaled (12 samples, 0.02%) + + + +genunix`kmem_cache_alloc (241 samples, 0.42%) + + + +FSS`fss_preempt (8 samples, 0.01%) + + + +genunix`fd_reserve (15 samples, 0.03%) + + + +genunix`cv_broadcast (16 samples, 0.03%) + + + +genunix`crgetmapped (57 samples, 0.10%) + + + +unix`mutex_exit (379 samples, 0.66%) + + + +unix`mutex_destroy (31 samples, 0.05%) + + + +lofs`table_lock_enter (189 samples, 0.33%) + + + +genunix`rwst_enter_common (264 samples, 0.46%) + + + +genunix`kmem_free (11 samples, 0.02%) + + + +unix`atomic_add_32 (134 samples, 0.23%) + + + +genunix`ufalloc (551 samples, 0.96%) + + + +genunix`audit_falloc (313 samples, 0.54%) + + + +lofs`lo_lookup (19,887 samples, 34.62%) +lofs`lo_lookup + + +unix`atomic_add_64 (110 samples, 0.19%) + + + +genunix`vn_vfsunlock (2,372 samples, 4.13%) +genu.. + + +genunix`openat (17 samples, 0.03%) + + + +unix`bcmp (45 samples, 0.08%) + + + +genunix`audit_getstate (62 samples, 0.11%) + + + +genunix`crfree (9 samples, 0.02%) + + + +genunix`kmem_cache_free (18 samples, 0.03%) + + + +genunix`vn_vfslocks_rele (903 samples, 1.57%) + + + +genunix`vn_invalid (20 samples, 0.03%) + + + +genunix`vn_vfslocks_rele (50 samples, 0.09%) + + + +genunix`lookuppnvp (10 samples, 0.02%) + + + +genunix`fd_find (161 samples, 0.28%) + + + +ufs`ufs_lookup (5,399 samples, 9.40%) +ufs`ufs_lookup + + +unix`0xfffffffffb800c7c (42 samples, 0.07%) + + + +genunix`vn_openat (14 samples, 0.02%) + + + +genunix`setf (16 samples, 0.03%) + + + +genunix`traverse (7,243 samples, 12.61%) +genunix`traverse + + +genunix`rwst_tryenter (734 samples, 1.28%) + + + +unix`mutex_enter (366 samples, 0.64%) + + + +genunix`fop_lookup (6,470 samples, 11.26%) +genunix`fop_lookup + + +unix`mutex_exit (135 samples, 0.24%) + + + +lofs`makelfsnode (82 samples, 0.14%) + + + +genunix`copen (7 samples, 0.01%) + + + diff --git a/benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz b/benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz new file mode 100644 index 00000000000..e7b762b88bb Binary files /dev/null and b/benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz differ diff --git a/benchmarks/cli/_script/flamegraph/example-perf.svg b/benchmarks/cli/_script/flamegraph/example-perf.svg new file mode 100644 index 00000000000..d4896fc3503 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/example-perf.svg @@ -0,0 +1,4895 @@ + + + + + + + + + + + + + +Flame Graph + +Reset Zoom +Search + + +rw_verify_area (9 samples, 0.68%) + + + +_raw_spin_lock_irqsave (2 samples, 0.15%) + + + +sun/nio/ch/FileDispatcherImpl:.read0 (31 samples, 2.36%) +s.. + + +do_sync_read (22 samples, 1.67%) + + + +sun/nio/ch/SocketChannelImpl:.write (209 samples, 15.89%) +sun/nio/ch/SocketChannel.. + + +timerqueue_del (1 samples, 0.08%) + + + +io/netty/channel/AdaptiveRecvByteBufAllocator$HandleImpl:.record (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptRuntime:.setObjectProp (86 samples, 6.54%) +org/mozi.. + + +read_tsc (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (14 samples, 1.06%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (45 samples, 3.42%) +org.. + + +netdev_pick_tx (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.write (33 samples, 2.51%) +io.. + + +java/lang/String:.equals (1 samples, 0.08%) + + + +system_call_fastpath (7 samples, 0.53%) + + + +GCTaskManager::get_task (1 samples, 0.08%) + + + +security_file_free (1 samples, 0.08%) + + + +apparmor_socket_recvmsg (5 samples, 0.38%) + + + +itable stub (1 samples, 0.08%) + + + +skb_release_data (3 samples, 0.23%) + + + +hrtimer_try_to_cancel (3 samples, 0.23%) + + + +default_wake_function (25 samples, 1.90%) +d.. + + +__remove_hrtimer (3 samples, 0.23%) + + + +epoll_ctl (1 samples, 0.08%) + + + +fsnotify (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.nameOrFunction (4 samples, 0.30%) + + + +tcp_clean_rtx_queue (1 samples, 0.08%) + + + +tcp_send_delayed_ack (5 samples, 0.38%) + + + +org/mozilla/javascript/ScriptRuntime:.nameOrFunction (1 samples, 0.08%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +tcp_v4_rcv (87 samples, 6.62%) +tcp_v4_rcv + + +aeProcessEvents (1 samples, 0.08%) + + + +org/mozilla/javascript/NativeJavaObject:.initMembers (1 samples, 0.08%) + + + +schedule_preempt_disabled (2 samples, 0.15%) + + + +kfree (1 samples, 0.08%) + + + +sun/nio/ch/SocketChannelImpl:.write (1 samples, 0.08%) + + + +sk_reset_timer (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (6 samples, 0.46%) + + + +remote_function (4 samples, 0.30%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.skipControlCharacters (1 samples, 0.08%) + + + +intel_pmu_enable_all (4 samples, 0.30%) + + + +mod_timer (5 samples, 0.38%) + + + +io/netty/handler/codec/MessageToMessageEncoder:.write (31 samples, 2.36%) +i.. + + +io/netty/buffer/UnpooledHeapByteBuf:.init (1 samples, 0.08%) + + + +intel_pmu_enable_all (4 samples, 0.30%) + + + +io/netty/channel/AbstractChannelHandlerContext:.write (2 samples, 0.15%) + + + +enqueue_hrtimer (1 samples, 0.08%) + + + +itable stub (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.setAttributes (12 samples, 0.91%) + + + +cpuidle_idle_call (6 samples, 0.46%) + + + +system_call (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.get (2 samples, 0.15%) + + + +[unknown] (6 samples, 0.46%) + + + +Monitor::IWait (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.bind (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (40 samples, 3.04%) +org.. + + +__wake_up_sync_key (3 samples, 0.23%) + + + +system_call_fastpath (1 samples, 0.08%) + + + +vfs_write (85 samples, 6.46%) +vfs_write + + +mod_timer (2 samples, 0.15%) + + + +rcu_sysidle_enter (1 samples, 0.08%) + + + +oopDesc* PSPromotionManager::copy_to_survivor_spacefalse (1 samples, 0.08%) + + + +__wake_up_common (2 samples, 0.15%) + + + +io/netty/channel/AbstractChannelHandlerContext:.fireChannelRead (637 samples, 48.44%) +io/netty/channel/AbstractChannelHandlerContext:.fireChannelRead + + +_raw_spin_lock_irqsave (2 samples, 0.15%) + + + +ScavengeRootsTask::do_it (1 samples, 0.08%) + + + +tcp_urg (1 samples, 0.08%) + + + +aa_file_perm (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.setObjectProp (21 samples, 1.60%) + + + +__remove_hrtimer (1 samples, 0.08%) + + + +put_filp (1 samples, 0.08%) + + + +skb_free_head (1 samples, 0.08%) + + + +apparmor_file_permission (1 samples, 0.08%) + + + +ktime_get (1 samples, 0.08%) + + + +JavaCalls::call_virtual (956 samples, 72.70%) +JavaCalls::call_virtual + + +__copy_skb_header (1 samples, 0.08%) + + + +__slab_alloc (1 samples, 0.08%) + + + +cpuidle_idle_call (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.has (30 samples, 2.28%) +o.. + + +ip_queue_xmit (51 samples, 3.88%) +ip_q.. + + +org/mozilla/javascript/NativeCall:.init (15 samples, 1.14%) + + + +tcp_ack (9 samples, 0.68%) + + + +sys_ioctl (5 samples, 0.38%) + + + +fsnotify (2 samples, 0.15%) + + + +sk_reset_timer (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.addKnownAbsentSlot (1 samples, 0.08%) + + + +lapic_next_deadline (2 samples, 0.15%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_Server2_js_1:.call (79 samples, 6.01%) +org/mozi.. + + +sys_execve (1 samples, 0.08%) + + + +perf_event_enable (5 samples, 0.38%) + + + +sys_futex (1 samples, 0.08%) + + + +java/lang/String:.init (1 samples, 0.08%) + + + +inet_recvmsg (7 samples, 0.53%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (2 samples, 0.15%) + + + +io/netty/util/internal/AppendableCharSequence:.substring (4 samples, 0.30%) + + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +x86_pmu_enable (4 samples, 0.30%) + + + +__libc_read (1 samples, 0.08%) + + + +tcp_sendmsg (77 samples, 5.86%) +tcp_sen.. + + +cpuidle_enter_state (12 samples, 0.91%) + + + +flush_tlb_mm_range (1 samples, 0.08%) + + + +ksize (1 samples, 0.08%) + + + +cpu_startup_entry (44 samples, 3.35%) +cpu.. + + +pthread_self (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +_raw_spin_lock_bh (1 samples, 0.08%) + + + +io/netty/channel/DefaultChannelPipeline$HeadContext:.flush (2 samples, 0.15%) + + + +tcp_rcv_established (23 samples, 1.75%) + + + +org/mozilla/javascript/BaseFunction:.execIdCall (48 samples, 3.65%) +org/.. + + +lapic_next_deadline (1 samples, 0.08%) + + + +[unknown] (197 samples, 14.98%) +[unknown] + + +io/netty/channel/AbstractChannelHandlerContext:.fireChannelReadComplete (242 samples, 18.40%) +io/netty/channel/AbstractCha.. + + +bictcp_cong_avoid (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.nameOrFunction (5 samples, 0.38%) + + + +JavaCalls::call_virtual (956 samples, 72.70%) +JavaCalls::call_virtual + + +resched_task (2 samples, 0.15%) + + + +sock_wfree (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (4 samples, 0.30%) + + + +io/netty/buffer/AbstractByteBuf:.getByte (1 samples, 0.08%) + + + +check_preempt_curr (2 samples, 0.15%) + + + +io/netty/channel/ChannelOutboundBuffer:.progress (1 samples, 0.08%) + + + +tcp_current_mss (1 samples, 0.08%) + + + +__execve (1 samples, 0.08%) + + + +hrtimer_force_reprogram (1 samples, 0.08%) + + + +__GI___mprotect (1 samples, 0.08%) + + + +ep_send_events_proc (9 samples, 0.68%) + + + +schedule (11 samples, 0.84%) + + + +org/mozilla/javascript/IdScriptableObject:.put (3 samples, 0.23%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (409 samples, 31.10%) +org/mozilla/javascript/gen/file__root_vert_x_2_1_.. + + +io/netty/channel/ChannelOutboundBuffer:.decrementPendingOutboundBytes (2 samples, 0.15%) + + + +ktime_get_real (1 samples, 0.08%) + + + +aa_revalidate_sk (2 samples, 0.15%) + + + +stats_record (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +__perf_event_enable (4 samples, 0.30%) + + + +__alloc_skb (9 samples, 0.68%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (17 samples, 1.29%) + + + +socket_readable (2 samples, 0.15%) + + + +ns_to_timeval (1 samples, 0.08%) + + + +ip_rcv (33 samples, 2.51%) +ip.. + + +SafepointSynchronize::begin (1 samples, 0.08%) + + + +java/nio/DirectByteBuffer:.duplicate (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (20 samples, 1.52%) + + + +io/netty/channel/nio/AbstractNioByteChannel$NioByteUnsafe:.read (939 samples, 71.41%) +io/netty/channel/nio/AbstractNioByteChannel$NioByteUnsafe:.read + + +raw_local_deliver (1 samples, 0.08%) + + + +__dev_queue_xmit (4 samples, 0.30%) + + + +skb_copy_datagram_iovec (3 samples, 0.23%) + + + +apic_timer_interrupt (1 samples, 0.08%) + + + +do_vfs_ioctl (5 samples, 0.38%) + + + +do_sync_read (8 samples, 0.61%) + + + +system_call_after_swapgs (1 samples, 0.08%) + + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +call_function_single_interrupt (4 samples, 0.30%) + + + +io/netty/handler/codec/http/HttpHeaders:.hash (4 samples, 0.30%) + + + +io/netty/handler/codec/http/DefaultHttpMessage:.init (2 samples, 0.15%) + + + +rcu_sysidle_enter (1 samples, 0.08%) + + + +java/nio/channels/spi/AbstractInterruptibleChannel:.end (3 samples, 0.23%) + + + +clockevents_program_event (2 samples, 0.15%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (9 samples, 0.68%) + + + +tcp_try_rmem_schedule (2 samples, 0.15%) + + + +__schedule (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (10 samples, 0.76%) + + + +tcp_v4_md5_lookup (1 samples, 0.08%) + + + +CardTableExtension::scavenge_contents_parallel (20 samples, 1.52%) + + + +aa_revalidate_sk (1 samples, 0.08%) + + + +__fsnotify_parent (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.get (7 samples, 0.53%) + + + +sk_reset_timer (5 samples, 0.38%) + + + +__schedule (2 samples, 0.15%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.init (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +io/netty/channel/nio/AbstractNioByteChannel:.doWrite (225 samples, 17.11%) +io/netty/channel/nio/Abstr.. + + +timerqueue_add (1 samples, 0.08%) + + + +_raw_spin_unlock_irqrestore (2 samples, 0.15%) + + + +try_to_wake_up (24 samples, 1.83%) +t.. + + +org/mozilla/javascript/IdScriptableObject:.setAttributes (4 samples, 0.30%) + + + +io/netty/handler/codec/http/HttpResponseEncoder:.acceptOutboundMessage (1 samples, 0.08%) + + + +rw_verify_area (2 samples, 0.15%) + + + +x86_pmu_commit_txn (4 samples, 0.30%) + + + +alloc_pages_current (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_streams_j (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.setAttributes (7 samples, 0.53%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +tcp_transmit_skb (1 samples, 0.08%) + + + +sock_aio_read.part.8 (7 samples, 0.53%) + + + +sys_read (28 samples, 2.13%) +s.. + + +org/mozilla/javascript/ScriptRuntime:.setObjectProp (28 samples, 2.13%) +o.. + + +JavaCalls::call_helper (956 samples, 72.70%) +JavaCalls::call_helper + + +ttwu_do_wakeup (1 samples, 0.08%) + + + +generic_smp_call_function_single_interrupt (4 samples, 0.30%) + + + +mutex_unlock (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpHeaders:.hash (2 samples, 0.15%) + + + +http_parser_execute (1 samples, 0.08%) + + + +mod_timer (5 samples, 0.38%) + + + +system_call_fastpath (1 samples, 0.08%) + + + +tcp_recvmsg (13 samples, 0.99%) + + + +__slab_alloc (1 samples, 0.08%) + + + +__alloc_skb (7 samples, 0.53%) + + + +clockevents_program_event (1 samples, 0.08%) + + + +vfs_read (18 samples, 1.37%) + + + +__internal_add_timer (1 samples, 0.08%) + + + +epoll_wait (1 samples, 0.08%) + + + +lock_sock_nested (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +native_write_msr_safe (4 samples, 0.30%) + + + +Interpreter (956 samples, 72.70%) +Interpreter + + +org/mozilla/javascript/ScriptableObject:.getBase (4 samples, 0.30%) + + + +dev_hard_start_xmit (9 samples, 0.68%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +ip_output (46 samples, 3.50%) +ip_.. + + +account_entity_enqueue (1 samples, 0.08%) + + + +itable stub (1 samples, 0.08%) + + + +ip_rcv (1 samples, 0.08%) + + + +io/netty/buffer/AbstractByteBuf:.writeBytes (5 samples, 0.38%) + + + +tcp_clean_rtx_queue (14 samples, 1.06%) + + + +io/netty/channel/AbstractChannelHandlerContext:.flush (1 samples, 0.08%) + + + +sys_read (21 samples, 1.60%) + + + +[unknown] (10 samples, 0.76%) + + + +java/util/concurrent/ConcurrentHashMap:.get (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannel:.hashCode (4 samples, 0.30%) + + + +rcu_idle_enter (1 samples, 0.08%) + + + +gettimeofday@plt (1 samples, 0.08%) + + + +__do_softirq (103 samples, 7.83%) +__do_softirq + + +org/mozilla/javascript/ScriptRuntime:.nameOrFunction (8 samples, 0.61%) + + + +io/netty/buffer/AbstractByteBuf:.writeBytes (3 samples, 0.23%) + + + +inotify_add_watch (1 samples, 0.08%) + + + +fdval (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpHeaders:.encode (7 samples, 0.53%) + + + +unsafe_arraycopy (1 samples, 0.08%) + + + +sk_stream_alloc_skb (10 samples, 0.76%) + + + +lock_timer_base.isra.35 (1 samples, 0.08%) + + + +ip_local_out (121 samples, 9.20%) +ip_local_out + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +io/netty/util/internal/AppendableCharSequence:.substring (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptRuntime:.bind (1 samples, 0.08%) + + + +ep_poll (53 samples, 4.03%) +ep_p.. + + +lock_hrtimer_base.isra.19 (1 samples, 0.08%) + + + +InstanceKlass::oop_push_contents (1 samples, 0.08%) + + + +cpuacct_charge (1 samples, 0.08%) + + + +harmonize_features.isra.92.part.93 (1 samples, 0.08%) + + + +update_rq_clock.part.63 (1 samples, 0.08%) + + + +native_write_msr_safe (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.findNonWhitespace (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_streams_j (1 samples, 0.08%) + + + +org/mozilla/javascript/BaseFunction:.construct (156 samples, 11.86%) +org/mozilla/javas.. + + +_raw_spin_lock (2 samples, 0.15%) + + + +cpu_function_call (5 samples, 0.38%) + + + +fget_light (2 samples, 0.15%) + + + +start_kernel (24 samples, 1.83%) +s.. + + +native_write_msr_safe (2 samples, 0.15%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (511 samples, 38.86%) +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io.. + + +ipv4_mtu (1 samples, 0.08%) + + + +__schedule (11 samples, 0.84%) + + + +system_call_fastpath (88 samples, 6.69%) +system_ca.. + + +io/netty/channel/nio/NioEventLoop:.select (7 samples, 0.53%) + + + +org/mozilla/javascript/IdScriptableObject:.get (3 samples, 0.23%) + + + +org/mozilla/javascript/TopLevel:.getBuiltinPrototype (7 samples, 0.53%) + + + +sun/nio/ch/IOUtil:.readIntoNativeBuffer (31 samples, 2.36%) +s.. + + +itable stub (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.write (6 samples, 0.46%) + + + +timerqueue_del (1 samples, 0.08%) + + + +__tcp_v4_send_check (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.nameOrFunction (3 samples, 0.23%) + + + +io/netty/buffer/AbstractByteBuf:.writeBytes (4 samples, 0.30%) + + + +org/mozilla/javascript/WrapFactory:.wrapAsJavaObject (1 samples, 0.08%) + + + +io/netty/handler/codec/http/DefaultHttpMessage:.init (2 samples, 0.15%) + + + +_raw_spin_lock_irqsave (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.put (7 samples, 0.53%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.add0 (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (8 samples, 0.61%) + + + +java/util/ArrayList:.ensureCapacityInternal (1 samples, 0.08%) + + + +__wake_up_locked (25 samples, 1.90%) +_.. + + +java/util/HashMap:.getNode (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.toObjectOrNull (1 samples, 0.08%) + + + +__tcp_push_pending_frames (1 samples, 0.08%) + + + +[unknown] (61 samples, 4.64%) +[unkn.. + + +__slab_free (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.put (11 samples, 0.84%) + + + +sock_def_readable (5 samples, 0.38%) + + + +gmain (1 samples, 0.08%) + + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +__kmalloc_reserve.isra.26 (3 samples, 0.23%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (5 samples, 0.38%) + + + +org/mozilla/javascript/NativeCall:.init (20 samples, 1.52%) + + + +org/mozilla/javascript/WrapFactory:.wrap (1 samples, 0.08%) + + + +_raw_spin_lock_bh (1 samples, 0.08%) + + + +aeProcessEvents (3 samples, 0.23%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.executor (1 samples, 0.08%) + + + +fget_light (2 samples, 0.15%) + + + +io/netty/buffer/PooledByteBufAllocator:.newDirectBuffer (2 samples, 0.15%) + + + +menu_select (1 samples, 0.08%) + + + +generic_smp_call_function_single_interrupt (4 samples, 0.30%) + + + +org/mozilla/javascript/TopLevel:.getBuiltinPrototype (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.setAttributes (1 samples, 0.08%) + + + +schedule_hrtimeout_range_clock (20 samples, 1.52%) + + + +io/netty/buffer/UnreleasableByteBuf:.duplicate (1 samples, 0.08%) + + + +tick_program_event (2 samples, 0.15%) + + + +__netif_receive_skb_core (33 samples, 2.51%) +__.. + + +java/util/HashMap:.getNode (1 samples, 0.08%) + + + +io/netty/buffer/AbstractByteBuf:.forEachByteAsc0 (2 samples, 0.15%) + + + +get_next_timer_interrupt (2 samples, 0.15%) + + + +vtable stub (1 samples, 0.08%) + + + +start_secondary (44 samples, 3.35%) +sta.. + + +skb_release_all (3 samples, 0.23%) + + + +update_cfs_rq_blocked_load (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +call_function_single_interrupt (4 samples, 0.30%) + + + +sun/nio/ch/SocketChannelImpl:.read (40 samples, 3.04%) +sun.. + + +sys_epoll_wait (1 samples, 0.08%) + + + +tcp_check_space (1 samples, 0.08%) + + + +__wake_up_common (25 samples, 1.90%) +_.. + + +native_sched_clock (1 samples, 0.08%) + + + +fget_light (3 samples, 0.23%) + + + +sys_inotify_add_watch (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.getValue (1 samples, 0.08%) + + + +_raw_spin_lock (1 samples, 0.08%) + + + +smp_call_function_single_interrupt (4 samples, 0.30%) + + + +__kmalloc_node_track_caller (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$RelinkedSlot:.getValue (1 samples, 0.08%) + + + +tcp_send_mss (6 samples, 0.46%) + + + +sched_clock (1 samples, 0.08%) + + + +kmem_cache_alloc_node (4 samples, 0.30%) + + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +tick_sched_handle.isra.17 (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getBase (1 samples, 0.08%) + + + +[unknown] (6 samples, 0.46%) + + + +io/netty/util/internal/AppendableCharSequence:.substring (2 samples, 0.15%) + + + +__inet_lookup_established (2 samples, 0.15%) + + + +apparmor_file_permission (1 samples, 0.08%) + + + +dst_release (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectEncoder:.encode (1 samples, 0.08%) + + + +open_exec (1 samples, 0.08%) + + + +tcp_transmit_skb (132 samples, 10.04%) +tcp_transmit_skb + + +ttwu_do_wakeup (5 samples, 0.38%) + + + +idle_cpu (1 samples, 0.08%) + + + +__lll_unlock_wake (1 samples, 0.08%) + + + +[unknown] (7 samples, 0.53%) + + + +security_file_permission (2 samples, 0.15%) + + + +[unknown] (1 samples, 0.08%) + + + +__switch_to (1 samples, 0.08%) + + + +io/netty/channel/DefaultChannelPromise:.trySuccess (3 samples, 0.23%) + + + +org/mozilla/javascript/IdScriptableObject:.has (7 samples, 0.53%) + + + +native_write_msr_safe (3 samples, 0.23%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.add0 (2 samples, 0.15%) + + + +do_softirq (103 samples, 7.83%) +do_softirq + + +rw_verify_area (1 samples, 0.08%) + + + +tcp_poll (1 samples, 0.08%) + + + +tcp_rearm_rto (5 samples, 0.38%) + + + +io/netty/channel/AbstractChannelHandlerContext:.newPromise (1 samples, 0.08%) + + + +tick_nohz_idle_exit (5 samples, 0.38%) + + + +org/mozilla/javascript/BaseFunction:.execIdCall (60 samples, 4.56%) +org/m.. + + +org/mozilla/javascript/ScriptableObject:.putImpl (1 samples, 0.08%) + + + +_copy_from_user (1 samples, 0.08%) + + + +__netif_receive_skb (34 samples, 2.59%) +__.. + + +java/util/concurrent/ConcurrentHashMap:.get (3 samples, 0.23%) + + + +fput (1 samples, 0.08%) + + + +JavaThread::thread_main_inner (956 samples, 72.70%) +JavaThread::thread_main_inner + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +io/netty/util/Recycler:.get (2 samples, 0.15%) + + + +[unknown] (6 samples, 0.46%) + + + +__dev_queue_xmit (1 samples, 0.08%) + + + +common_file_perm (1 samples, 0.08%) + + + +org/mozilla/javascript/JavaMembers:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.getPropFunctionAndThisHelper (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (6 samples, 0.46%) + + + +jiffies_to_timeval (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.setName (2 samples, 0.15%) + + + +PSRootsClosurefalse::do_oop (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +vtable stub (1 samples, 0.08%) + + + +skb_clone (4 samples, 0.30%) + + + +OldToYoungRootsTask::do_it (20 samples, 1.52%) + + + +io/netty/channel/ChannelDuplexHandler:.flush (237 samples, 18.02%) +io/netty/channel/ChannelDupl.. + + +org/mozilla/javascript/ScriptableObject:.putImpl (1 samples, 0.08%) + + + +mutex_unlock (1 samples, 0.08%) + + + +hrtimer_force_reprogram (1 samples, 0.08%) + + + +stub_execve (1 samples, 0.08%) + + + +sock_poll (3 samples, 0.23%) + + + +org/mozilla/javascript/IdScriptableObject:.setAttributes (5 samples, 0.38%) + + + +_raw_spin_lock_bh (1 samples, 0.08%) + + + +native_write_msr_safe (3 samples, 0.23%) + + + +sched_clock_cpu (1 samples, 0.08%) + + + +io/netty/channel/DefaultChannelPipeline$HeadContext:.flush (232 samples, 17.64%) +io/netty/channel/DefaultCha.. + + +org/mozilla/javascript/IdScriptableObject:.get (4 samples, 0.30%) + + + +rcu_idle_enter (1 samples, 0.08%) + + + +java (995 samples, 75.67%) +java + + +tcp_cleanup_rbuf (2 samples, 0.15%) + + + +org/mozilla/javascript/NativeJavaObject:.initMembers (4 samples, 0.30%) + + + +org/mozilla/javascript/NativeCall:.init (16 samples, 1.22%) + + + +http_parser_execute (2 samples, 0.15%) + + + +_raw_spin_unlock_irqrestore (1 samples, 0.08%) + + + +ThreadRootsTask::do_it (3 samples, 0.23%) + + + +mutex_lock (3 samples, 0.23%) + + + +cpu_startup_entry (23 samples, 1.75%) + + + +itable stub (1 samples, 0.08%) + + + +__srcu_read_lock (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (5 samples, 0.38%) + + + +org/vertx/java/core/impl/DefaultVertx:.setContext (1 samples, 0.08%) + + + +ip_rcv_finish (89 samples, 6.77%) +ip_rcv_fi.. + + +response_complete (13 samples, 0.99%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.skipControlCharacters (2 samples, 0.15%) + + + +tcp_v4_rcv (27 samples, 2.05%) +t.. + + +ktime_get_ts (2 samples, 0.15%) + + + +tick_nohz_restart (4 samples, 0.30%) + + + +io/netty/channel/ChannelOutboundHandlerAdapter:.flush (1 samples, 0.08%) + + + +sun/nio/ch/FileDispatcherImpl:.write0 (2 samples, 0.15%) + + + +GCTaskThread::run (28 samples, 2.13%) +G.. + + +io/netty/handler/codec/http/HttpHeaders:.encode (1 samples, 0.08%) + + + +__kmalloc_node_track_caller (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.put (25 samples, 1.90%) +o.. + + +org/mozilla/javascript/IdScriptableObject:.has (2 samples, 0.15%) + + + +atomic_notifier_call_chain (1 samples, 0.08%) + + + +remote_function (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +common_file_perm (1 samples, 0.08%) + + + +sun/nio/ch/SocketChannelImpl:.isConnected (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.has (9 samples, 0.68%) + + + +tcp_init_tso_segs (1 samples, 0.08%) + + + +org/mozilla/javascript/BaseFunction:.findInstanceIdInfo (1 samples, 0.08%) + + + +tcp_v4_do_rcv (77 samples, 5.86%) +tcp_v4_.. + + +__tcp_push_pending_frames (61 samples, 4.64%) +__tcp.. + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +native_read_tsc (1 samples, 0.08%) + + + +tcp_md5_do_lookup (1 samples, 0.08%) + + + +do_sync_write (186 samples, 14.14%) +do_sync_write + + +cpuidle_enter_state (4 samples, 0.30%) + + + +ep_poll_callback (1 samples, 0.08%) + + + +x86_pmu_enable (4 samples, 0.30%) + + + +copy_user_generic_string (3 samples, 0.23%) + + + +perf_pmu_enable (4 samples, 0.30%) + + + +vfs_read (25 samples, 1.90%) +v.. + + +x86_64_start_reservations (24 samples, 1.83%) +x.. + + +security_file_permission (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.addKnownAbsentSlot (1 samples, 0.08%) + + + +nr_iowait_cpu (1 samples, 0.08%) + + + +__hrtimer_start_range_ns (2 samples, 0.15%) + + + +system_call_after_swapgs (1 samples, 0.08%) + + + +release_sock (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (11 samples, 0.84%) + + + +_raw_spin_unlock_irqrestore (1 samples, 0.08%) + + + +itable stub (1 samples, 0.08%) + + + +call_stub (956 samples, 72.70%) +call_stub + + +dev_hard_start_xmit (3 samples, 0.23%) + + + +dev_queue_xmit (11 samples, 0.84%) + + + +task_nice (2 samples, 0.15%) + + + +ip_finish_output (119 samples, 9.05%) +ip_finish_out.. + + +__remove_hrtimer (1 samples, 0.08%) + + + +sys_epoll_wait (4 samples, 0.30%) + + + +rcu_cpu_has_callbacks (1 samples, 0.08%) + + + +java/lang/ThreadLocal:.get (1 samples, 0.08%) + + + +rcu_idle_exit (1 samples, 0.08%) + + + +net_rx_action (97 samples, 7.38%) +net_rx_act.. + + +lock_sock_nested (1 samples, 0.08%) + + + +mod_timer (2 samples, 0.15%) + + + +apparmor_file_free_security (1 samples, 0.08%) + + + +__remove_hrtimer (1 samples, 0.08%) + + + +tcp_established_options (4 samples, 0.30%) + + + +sk_reset_timer (5 samples, 0.38%) + + + +io/netty/channel/ChannelOutboundHandlerAdapter:.flush (235 samples, 17.87%) +io/netty/channel/ChannelOut.. + + +org/mozilla/javascript/NativeFunction:.initScriptFunction (1 samples, 0.08%) + + + +menu_reflect (1 samples, 0.08%) + + + +__slab_alloc (3 samples, 0.23%) + + + +PSScavengeKlassClosure::do_klass (1 samples, 0.08%) + + + +__ip_local_out (1 samples, 0.08%) + + + +org/mozilla/javascript/TopLevel:.getBuiltinPrototype (5 samples, 0.38%) + + + +tcp_send_delayed_ack (3 samples, 0.23%) + + + +arch_local_irq_save (1 samples, 0.08%) + + + +__kmalloc_node_track_caller (1 samples, 0.08%) + + + +java/nio/channels/spi/AbstractInterruptibleChannel:.end (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (15 samples, 1.14%) + + + +apparmor_file_permission (2 samples, 0.15%) + + + +mutex_lock (2 samples, 0.15%) + + + +sys_epoll_ctl (5 samples, 0.38%) + + + +hrtimer_interrupt (1 samples, 0.08%) + + + +ParallelTaskTerminator::offer_termination (2 samples, 0.15%) + + + +dequeue_entity (4 samples, 0.30%) + + + +io/netty/buffer/PooledByteBuf:.deallocate (5 samples, 0.38%) + + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +wrk (240 samples, 18.25%) +wrk + + +perf_pmu_enable (4 samples, 0.30%) + + + +org/mozilla/javascript/IdScriptableObject:.get (2 samples, 0.15%) + + + +remote_function (4 samples, 0.30%) + + + +__GI___ioctl (5 samples, 0.38%) + + + +socket_readable (2 samples, 0.15%) + + + +epoll_ctl (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (21 samples, 1.60%) + + + +tick_nohz_stop_sched_tick (4 samples, 0.30%) + + + +io/netty/channel/DefaultChannelPipeline$HeadContext:.write (6 samples, 0.46%) + + + +tcp_write_xmit (147 samples, 11.18%) +tcp_write_xmit + + +org/mozilla/javascript/ScriptRuntime:.getPropFunctionAndThisHelper (5 samples, 0.38%) + + + +lock_hrtimer_base.isra.19 (1 samples, 0.08%) + + + +inet_recvmsg (17 samples, 1.29%) + + + +native_write_msr_safe (4 samples, 0.30%) + + + +ip_local_out (46 samples, 3.50%) +ip_.. + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.has (1 samples, 0.08%) + + + +local_bh_enable (42 samples, 3.19%) +loc.. + + +hrtimer_start_range_ns (3 samples, 0.23%) + + + +jlong_disjoint_arraycopy (1 samples, 0.08%) + + + +ep_send_events_proc (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getParentScope (3 samples, 0.23%) + + + +itable stub (1 samples, 0.08%) + + + +path_openat (1 samples, 0.08%) + + + +activate_task (7 samples, 0.53%) + + + +pick_next_task_fair (1 samples, 0.08%) + + + +security_file_permission (5 samples, 0.38%) + + + +io/netty/channel/ChannelOutboundBuffer:.decrementPendingOutboundBytes (1 samples, 0.08%) + + + +system_call_fastpath (56 samples, 4.26%) +syste.. + + +org/mozilla/javascript/NativeFunction:.initScriptFunction (6 samples, 0.46%) + + + +ip_local_deliver_finish (30 samples, 2.28%) +i.. + + +sock_read (2 samples, 0.15%) + + + +deactivate_task (7 samples, 0.53%) + + + +lock_sock_nested (1 samples, 0.08%) + + + +sock_put (1 samples, 0.08%) + + + +mod_timer (3 samples, 0.23%) + + + +aeProcessEvents (171 samples, 13.00%) +aeProcessEvents + + +io/netty/buffer/AbstractByteBuf:.ensureWritable (2 samples, 0.15%) + + + +tick_nohz_stop_sched_tick (5 samples, 0.38%) + + + +org/mozilla/javascript/NativeJavaMethod:.findCachedFunction (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpMethod:.valueOf (2 samples, 0.15%) + + + +hrtimer_try_to_cancel (1 samples, 0.08%) + + + +system_call (1 samples, 0.08%) + + + +hrtimer_cancel (1 samples, 0.08%) + + + +system_call_fastpath (196 samples, 14.90%) +system_call_fastpath + + +io/netty/channel/AbstractChannelHandlerContext:.read (2 samples, 0.15%) + + + +[unknown] (10 samples, 0.76%) + + + +io/netty/handler/codec/MessageToMessageEncoder:.write (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.put (47 samples, 3.57%) +org.. + + +jlong_disjoint_arraycopy (1 samples, 0.08%) + + + +[unknown] (1 samples, 0.08%) + + + +native_read_tsc (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.splitHeader (8 samples, 0.61%) + + + +intel_pmu_enable_all (4 samples, 0.30%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.set (3 samples, 0.23%) + + + +read_tsc (1 samples, 0.08%) + + + +_raw_spin_lock (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.toObjectOrNull (1 samples, 0.08%) + + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +dequeue_task_fair (6 samples, 0.46%) + + + +org/mozilla/javascript/ScriptRuntime:.getPropFunctionAndThisHelper (1 samples, 0.08%) + + + +do_softirq (38 samples, 2.89%) +do.. + + +response_complete (2 samples, 0.15%) + + + +get_next_timer_interrupt (3 samples, 0.23%) + + + +__perf_event_enable (4 samples, 0.30%) + + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +lapic_next_deadline (3 samples, 0.23%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (4 samples, 0.30%) + + + +fput (1 samples, 0.08%) + + + +tcp_rearm_rto (3 samples, 0.23%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (2 samples, 0.15%) + + + +group_sched_in (4 samples, 0.30%) + + + +__getnstimeofday (1 samples, 0.08%) + + + +java/util/Arrays:.copyOf (1 samples, 0.08%) + + + +local_bh_enable (104 samples, 7.91%) +local_bh_en.. + + +tcp_event_new_data_sent (3 samples, 0.23%) + + + +read_tsc (2 samples, 0.15%) + + + +system_call_fastpath (6 samples, 0.46%) + + + +tcp_prequeue (1 samples, 0.08%) + + + +call_function_single_interrupt (4 samples, 0.30%) + + + +get_page_from_freelist (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.setAttributes (5 samples, 0.38%) + + + +io/netty/channel/AbstractChannelHandlerContext:.read (4 samples, 0.30%) + + + +org/vertx/java/core/http/impl/VertxHttpHandler:.write (34 samples, 2.59%) +or.. + + +_raw_spin_lock_irqsave (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.splitInitialLine (5 samples, 0.38%) + + + +org/mozilla/javascript/ScriptableObject:.getParentScope (4 samples, 0.30%) + + + +org/mozilla/javascript/IdScriptableObject:.get (3 samples, 0.23%) + + + +org/mozilla/javascript/ScriptRuntime:.getObjectProp (1 samples, 0.08%) + + + +swapper (72 samples, 5.48%) +swapper + + +ktime_get (1 samples, 0.08%) + + + +_raw_spin_unlock_irqrestore (1 samples, 0.08%) + + + +java/lang/ThreadLocal:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (416 samples, 31.63%) +org/mozilla/javascript/gen/file__root_vert_x_2_1_5.. + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +__schedule (4 samples, 0.30%) + + + +bictcp_cong_avoid (3 samples, 0.23%) + + + +tcp_rcv_space_adjust (2 samples, 0.15%) + + + +JavaThread::run (956 samples, 72.70%) +JavaThread::run + + +apparmor_socket_sendmsg (1 samples, 0.08%) + + + +InstanceKlass::oop_push_contents (8 samples, 0.61%) + + + +sun/nio/ch/SocketChannelImpl:.isConnected (1 samples, 0.08%) + + + +__libc_start_main (6 samples, 0.46%) + + + +tcp_is_cwnd_limited (2 samples, 0.15%) + + + +sun/nio/ch/FileDispatcherImpl:.write0 (203 samples, 15.44%) +sun/nio/ch/FileDispatch.. + + +internal_add_timer (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.setAttributes (2 samples, 0.15%) + + + +[unknown] (30 samples, 2.28%) +[.. + + +io/netty/buffer/AbstractByteBufAllocator:.heapBuffer (3 samples, 0.23%) + + + +__tcp_push_pending_frames (149 samples, 11.33%) +__tcp_push_pendi.. + + +ClassLoaderDataGraph::oops_do (1 samples, 0.08%) + + + +tick_nohz_stop_idle (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.executor (1 samples, 0.08%) + + + +__skb_clone (1 samples, 0.08%) + + + +tcp_ack (20 samples, 1.52%) + + + +__inet_lookup_established (4 samples, 0.30%) + + + +org/mozilla/javascript/NativeJavaMethod:.findFunction (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (4 samples, 0.30%) + + + +enqueue_task (7 samples, 0.53%) + + + +sock_def_readable (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.addKnownAbsentSlot (1 samples, 0.08%) + + + +tcp_data_queue (39 samples, 2.97%) +tc.. + + +security_file_permission (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.executor (1 samples, 0.08%) + + + +sun/reflect/DelegatingMethodAccessorImpl:.invoke (66 samples, 5.02%) +sun/re.. + + +__skb_clone (1 samples, 0.08%) + + + +org/vertx/java/platform/impl/RhinoContextFactory:.onContextCreated (1 samples, 0.08%) + + + +tcp_poll (1 samples, 0.08%) + + + +netif_skb_dev_features (1 samples, 0.08%) + + + +ep_scan_ready_list.isra.9 (4 samples, 0.30%) + + + +native_write_msr_safe (4 samples, 0.30%) + + + +copy_user_generic_string (1 samples, 0.08%) + + + +intel_pmu_enable_all (4 samples, 0.30%) + + + +__switch_to (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.has (4 samples, 0.30%) + + + +__hrtimer_start_range_ns (3 samples, 0.23%) + + + +__srcu_read_lock (2 samples, 0.15%) + + + +io/netty/channel/AbstractChannelHandlerContext:.validatePromise (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (11 samples, 0.84%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (8 samples, 0.61%) + + + +org/mozilla/javascript/NativeJavaMethod:.findCachedFunction (2 samples, 0.15%) + + + +sock_poll (1 samples, 0.08%) + + + +tick_program_event (3 samples, 0.23%) + + + +tcp_transmit_skb (55 samples, 4.18%) +tcp_.. + + +org/mozilla/javascript/NativeFunction:.initScriptFunction (1 samples, 0.08%) + + + +org/mozilla/javascript/WrapFactory:.wrap (5 samples, 0.38%) + + + +java/lang/String:.getBytes (3 samples, 0.23%) + + + +org/mozilla/javascript/NativeJavaObject:.initMembers (4 samples, 0.30%) + + + +bictcp_cong_avoid (1 samples, 0.08%) + + + +ktime_get_real (3 samples, 0.23%) + + + +java/lang/ThreadLocal:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (21 samples, 1.60%) + + + +rcu_sysidle_force_exit (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.getValue (2 samples, 0.15%) + + + +aa_file_perm (1 samples, 0.08%) + + + +tick_sched_timer (1 samples, 0.08%) + + + +sk_reset_timer (3 samples, 0.23%) + + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +msecs_to_jiffies (1 samples, 0.08%) + + + +ipv4_dst_check (1 samples, 0.08%) + + + +tcp_write_xmit (60 samples, 4.56%) +tcp_w.. + + +io/netty/util/Recycler:.recycle (1 samples, 0.08%) + + + +group_sched_in (4 samples, 0.30%) + + + +generic_exec_single (1 samples, 0.08%) + + + +menu_select (2 samples, 0.15%) + + + +org/mozilla/javascript/BaseFunction:.construct (1 samples, 0.08%) + + + +process_backlog (97 samples, 7.38%) +process_ba.. + + +__pthread_disable_asynccancel (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.decode (57 samples, 4.33%) +io/ne.. + + +schedule_preempt_disabled (4 samples, 0.30%) + + + +rcu_idle_exit (2 samples, 0.15%) + + + +tcp_send_mss (1 samples, 0.08%) + + + +[unknown] (26 samples, 1.98%) +[.. + + +org/mozilla/javascript/ScriptRuntime:.nameOrFunction (1 samples, 0.08%) + + + +inet_sendmsg (78 samples, 5.93%) +inet_se.. + + +__getnstimeofday (1 samples, 0.08%) + + + +kfree_skbmem (1 samples, 0.08%) + + + +smp_apic_timer_interrupt (1 samples, 0.08%) + + + +kmalloc_slab (2 samples, 0.15%) + + + +[unknown] (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.readHeaders (2 samples, 0.15%) + + + +ip_rcv_finish (32 samples, 2.43%) +ip.. + + +io/netty/channel/DefaultChannelPipeline$HeadContext:.read (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (8 samples, 0.61%) + + + +inet_ehashfn (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (33 samples, 2.51%) +or.. + + +frame::oops_do_internal (1 samples, 0.08%) + + + +thread_entry (956 samples, 72.70%) +thread_entry + + +sun/nio/ch/SelectorImpl:.select (7 samples, 0.53%) + + + +_raw_spin_lock_irq (1 samples, 0.08%) + + + +ttwu_do_activate.constprop.74 (12 samples, 0.91%) + + + +skb_copy_datagram_iovec (1 samples, 0.08%) + + + +Interpreter (956 samples, 72.70%) +Interpreter + + +io/netty/handler/codec/http/HttpObjectDecoder:.readHeaders (22 samples, 1.67%) + + + +org/vertx/java/core/http/impl/DefaultHttpServer$ServerHandler:.doMessageReceived (540 samples, 41.06%) +org/vertx/java/core/http/impl/DefaultHttpServer$ServerHandler:.doM.. + + +_raw_spin_unlock_irqrestore (1 samples, 0.08%) + + + +fget_light (1 samples, 0.08%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.contains (1 samples, 0.08%) + + + +kfree_skbmem (1 samples, 0.08%) + + + +__alloc_pages_nodemask (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.addKnownAbsentSlot (1 samples, 0.08%) + + + +enqueue_task_fair (5 samples, 0.38%) + + + +perf_pmu_enable (4 samples, 0.30%) + + + +tcp_rcv_established (73 samples, 5.55%) +tcp_rcv.. + + +org/mozilla/javascript/NativeJavaObject:.initMembers (1 samples, 0.08%) + + + +vfs_write (192 samples, 14.60%) +vfs_write + + +fdval (1 samples, 0.08%) + + + +ip_queue_xmit (122 samples, 9.28%) +ip_queue_xmit + + +sock_aio_write (82 samples, 6.24%) +sock_aio.. + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_streams_j (1 samples, 0.08%) + + + +aeMain (236 samples, 17.95%) +aeMain + + +io/netty/channel/ChannelDuplexHandler:.read (3 samples, 0.23%) + + + +org/vertx/java/core/http/impl/AssembledFullHttpResponse:.toLastContent (2 samples, 0.15%) + + + +internal_add_timer (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +vtable stub (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.getParentScope (1 samples, 0.08%) + + + +io/netty/buffer/PooledByteBufAllocator:.newDirectBuffer (2 samples, 0.15%) + + + +SpinPause (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.has (3 samples, 0.23%) + + + +java/nio/charset/CharsetEncoder:.replaceWith (2 samples, 0.15%) + + + +tcp_queue_rcv (2 samples, 0.15%) + + + +stats_record (3 samples, 0.23%) + + + +org/mozilla/javascript/WrapFactory:.wrap (5 samples, 0.38%) + + + +__wake_up_sync_key (27 samples, 2.05%) +_.. + + +__acct_update_integrals (1 samples, 0.08%) + + + +fget_light (1 samples, 0.08%) + + + +local_bh_enable (1 samples, 0.08%) + + + +eth_type_trans (1 samples, 0.08%) + + + +org/vertx/java/core/net/impl/VertxHandler:.channelRead (555 samples, 42.21%) +org/vertx/java/core/net/impl/VertxHandler:.channelRead + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +sock_put (1 samples, 0.08%) + + + +__kfree_skb (3 samples, 0.23%) + + + +dequeue_task (7 samples, 0.53%) + + + +io/netty/channel/AbstractChannelHandlerContext:.executor (1 samples, 0.08%) + + + +ip_rcv_finish (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.getObjectProp (4 samples, 0.30%) + + + +__tick_nohz_idle_enter (4 samples, 0.30%) + + + +__tcp_ack_snd_check (3 samples, 0.23%) + + + +[unknown] (4 samples, 0.30%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/NativeFunction:.initScriptFunction (2 samples, 0.15%) + + + +int_sqrt (1 samples, 0.08%) + + + +nmethod::fix_oop_relocations (1 samples, 0.08%) + + + +tcp_sendmsg (1 samples, 0.08%) + + + +java/lang/ThreadLocal:.get (1 samples, 0.08%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +org/mozilla/javascript/BaseFunction:.findPrototypeId (1 samples, 0.08%) + + + +native_write_msr_safe (3 samples, 0.23%) + + + +perf_pmu_enable (4 samples, 0.30%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (77 samples, 5.86%) +org/moz.. + + +__netif_receive_skb_core (94 samples, 7.15%) +__netif_r.. + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +hrtimer_start (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptRuntime:.setName (5 samples, 0.38%) + + + +__netif_receive_skb (94 samples, 7.15%) +__netif_r.. + + +change_protection (1 samples, 0.08%) + + + +io/netty/channel/ChannelOutboundBuffer:.current (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.flush (233 samples, 17.72%) +io/netty/channel/AbstractCh.. + + +do_softirq_own_stack (103 samples, 7.83%) +do_softirq_.. + + +do_filp_open (1 samples, 0.08%) + + + +x86_pmu_commit_txn (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +io/netty/util/Recycler:.get (1 samples, 0.08%) + + + +_raw_spin_lock (1 samples, 0.08%) + + + +sun/nio/ch/SocketChannelImpl:.isConnected (1 samples, 0.08%) + + + +change_protection_range (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (12 samples, 0.91%) + + + +__libc_write (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (513 samples, 39.01%) +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_.. + + +raw_local_deliver (1 samples, 0.08%) + + + +apparmor_file_permission (1 samples, 0.08%) + + + +VMThread::loop (1 samples, 0.08%) + + + +_raw_spin_unlock_irqrestore (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.newPromise (1 samples, 0.08%) + + + +epoll_ctl (7 samples, 0.53%) + + + +io/netty/handler/codec/http/HttpVersion:.compareTo (1 samples, 0.08%) + + + +io/netty/channel/nio/NioEventLoop:.processSelectedKeys (949 samples, 72.17%) +io/netty/channel/nio/NioEventLoop:.processSelectedKeys + + +io/netty/handler/codec/http/DefaultHttpHeaders:.init (1 samples, 0.08%) + + + +java/lang/String:.hashCode (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +effective_load.isra.35 (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (2 samples, 0.15%) + + + +rcu_sysidle_exit (1 samples, 0.08%) + + + +native_load_tls (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +org/mozilla/javascript/BaseFunction:.findPrototypeId (1 samples, 0.08%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_streams_j (6 samples, 0.46%) + + + +system_call_fastpath (22 samples, 1.67%) + + + +org/mozilla/javascript/TopLevel:.getBuiltinPrototype (1 samples, 0.08%) + + + +tcp_current_mss (5 samples, 0.38%) + + + +io/netty/channel/ChannelOutboundBuffer:.incrementPendingOutboundBytes (1 samples, 0.08%) + + + +oopDesc* PSPromotionManager::copy_to_survivor_spacefalse (2 samples, 0.15%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (156 samples, 11.86%) +org/mozilla/javas.. + + +StealTask::do_it (3 samples, 0.23%) + + + +Interpreter (956 samples, 72.70%) +Interpreter + + +PSPromotionManager::drain_stacks_depth (2 samples, 0.15%) + + + +sock_def_readable (32 samples, 2.43%) +so.. + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +perf (6 samples, 0.46%) + + + +__wake_up_common (27 samples, 2.05%) +_.. + + +org/mozilla/javascript/IdScriptableObject:.has (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpObjectDecoder$HeaderParser:.process (1 samples, 0.08%) + + + +common_file_perm (1 samples, 0.08%) + + + +io/netty/buffer/AbstractReferenceCountedByteBuf:.release (5 samples, 0.38%) + + + +hrtimer_force_reprogram (3 samples, 0.23%) + + + +org/vertx/java/core/impl/DefaultVertx:.setContext (1 samples, 0.08%) + + + +msecs_to_jiffies (1 samples, 0.08%) + + + +arch_cpu_idle (7 samples, 0.53%) + + + +[unknown] (91 samples, 6.92%) +[unknown] + + +tcp_cleanup_rbuf (1 samples, 0.08%) + + + +release_sock (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.flush (235 samples, 17.87%) +io/netty/channel/AbstractCh.. + + +io/netty/channel/AbstractChannelHandlerContext:.fireChannelReadComplete (1 samples, 0.08%) + + + +fput (2 samples, 0.15%) + + + +bictcp_acked (1 samples, 0.08%) + + + +java/nio/DirectByteBuffer:.duplicate (1 samples, 0.08%) + + + +io/netty/buffer/AbstractByteBuf:.checkIndex (3 samples, 0.23%) + + + +java/util/HashMap:.getNode (2 samples, 0.15%) + + + +ttwu_stat (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpHeaders:.hash (1 samples, 0.08%) + + + +org/vertx/java/core/net/impl/ConnectionBase:.write (38 samples, 2.89%) +or.. + + +local_apic_timer_interrupt (1 samples, 0.08%) + + + +inet_ehashfn (1 samples, 0.08%) + + + +__srcu_read_unlock (1 samples, 0.08%) + + + +java/nio/channels/spi/AbstractInterruptibleChannel:.begin (1 samples, 0.08%) + + + +skb_clone (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.fireChannelRead (562 samples, 42.74%) +io/netty/channel/AbstractChannelHandlerContext:.fireChannelRead + + +ip_local_deliver (89 samples, 6.77%) +ip_local_.. + + +org/mozilla/javascript/ScriptableObject:.createSlot (8 samples, 0.61%) + + + +itable stub (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +__do_softirq (36 samples, 2.74%) +__.. + + +io/netty/handler/codec/http/HttpMethod:.valueOf (2 samples, 0.15%) + + + +clockevents_program_event (3 samples, 0.23%) + + + +tcp_set_skb_tso_segs (1 samples, 0.08%) + + + +io/netty/buffer/AbstractByteBuf:.checkSrcIndex (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpVersion:.compareTo (2 samples, 0.15%) + + + +ttwu_do_activate.constprop.74 (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.setAttributes (6 samples, 0.46%) + + + +cpuidle_idle_call (21 samples, 1.60%) + + + +__hrtimer_start_range_ns (2 samples, 0.15%) + + + +io/netty/channel/ChannelOutboundHandlerAdapter:.read (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptRuntime:.bind (7 samples, 0.53%) + + + +x86_pmu_enable (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +sun/nio/ch/EPollArrayWrapper:.poll (5 samples, 0.38%) + + + +org/mozilla/javascript/IdScriptableObject:.setAttributes (12 samples, 0.91%) + + + +sock_read (3 samples, 0.23%) + + + +HandleArea::oops_do (1 samples, 0.08%) + + + +tcp_v4_send_check (1 samples, 0.08%) + + + +tcp_wfree (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$Slot:.getValue (1 samples, 0.08%) + + + +sun/nio/ch/FileDispatcherImpl:.read0 (1 samples, 0.08%) + + + +org/vertx/java/core/net/impl/VertxHandler:.channelReadComplete (240 samples, 18.25%) +org/vertx/java/core/net/impl.. + + +rcu_bh_qs (1 samples, 0.08%) + + + +lock_timer_base.isra.35 (1 samples, 0.08%) + + + +io/netty/buffer/PooledUnsafeDirectByteBuf:.setBytes (42 samples, 3.19%) +io/.. + + +sun/nio/cs/UTF_8$Encoder:.init (3 samples, 0.23%) + + + +io/netty/channel/ChannelDuplexHandler:.read (1 samples, 0.08%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +account_entity_dequeue (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.executor (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (17 samples, 1.29%) + + + +io/netty/handler/codec/http/HttpObjectEncoder:.encode (17 samples, 1.29%) + + + +org/mozilla/javascript/ScriptableObject:.addKnownAbsentSlot (2 samples, 0.15%) + + + +ksize (1 samples, 0.08%) + + + +[unknown] (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptRuntime:.toObjectOrNull (2 samples, 0.15%) + + + +fget_light (1 samples, 0.08%) + + + +sched_clock_idle_sleep_event (1 samples, 0.08%) + + + +sock_aio_write (185 samples, 14.07%) +sock_aio_write + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_streams_j (1 samples, 0.08%) + + + +smp_call_function_single (5 samples, 0.38%) + + + +org/mozilla/javascript/TopLevel:.getBuiltinPrototype (1 samples, 0.08%) + + + +ip_rcv (91 samples, 6.92%) +ip_rcv + + +tcp_sendmsg (176 samples, 13.38%) +tcp_sendmsg + + +release_sock (1 samples, 0.08%) + + + +ep_poll_callback (27 samples, 2.05%) +e.. + + +update_min_vruntime (1 samples, 0.08%) + + + +java/lang/Integer:.toString (1 samples, 0.08%) + + + +itable stub (1 samples, 0.08%) + + + +do_softirq_own_stack (37 samples, 2.81%) +do.. + + +io/netty/buffer/AbstractByteBufAllocator:.heapBuffer (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (3 samples, 0.23%) + + + +org/mozilla/javascript/ScriptableObject:.addKnownAbsentSlot (1 samples, 0.08%) + + + +sched_clock_cpu (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpHeaders:.encodeAscii0 (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptRuntime:.toObjectOrNull (1 samples, 0.08%) + + + +inet_sendmsg (1 samples, 0.08%) + + + +VMThread::run (1 samples, 0.08%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.init (1 samples, 0.08%) + + + +java/util/HashMap:.getNode (2 samples, 0.15%) + + + +__run_hrtimer (1 samples, 0.08%) + + + +java/nio/channels/spi/AbstractInterruptibleChannel:.begin (1 samples, 0.08%) + + + +io/netty/handler/codec/http/HttpHeaders:.hash (1 samples, 0.08%) + + + +sun/nio/ch/EPollSelectorImpl:.updateSelectedKeys (1 samples, 0.08%) + + + +x86_pmu_enable (4 samples, 0.30%) + + + +thread_main (237 samples, 18.02%) +thread_main + + +enqueue_hrtimer (1 samples, 0.08%) + + + +ep_poll (4 samples, 0.30%) + + + +sock_aio_read (7 samples, 0.53%) + + + +io/netty/buffer/AbstractByteBuf:.checkSrcIndex (3 samples, 0.23%) + + + +sock_aio_read (22 samples, 1.67%) + + + +io/netty/handler/codec/http/HttpObjectDecoder$LineParser:.parse (6 samples, 0.46%) + + + +sys_epoll_ctl (5 samples, 0.38%) + + + +java/lang/String:.init (4 samples, 0.30%) + + + +rb_erase (1 samples, 0.08%) + + + +select_estimate_accuracy (5 samples, 0.38%) + + + +link_path_walk (1 samples, 0.08%) + + + +sock_aio_read.part.8 (22 samples, 1.67%) + + + +local_bh_enable_ip (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (3 samples, 0.23%) + + + +__tcp_select_window (1 samples, 0.08%) + + + +fget_light (1 samples, 0.08%) + + + +io/netty/buffer/AbstractReferenceCountedByteBuf:.release (4 samples, 0.30%) + + + +acct_account_cputime (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.put (1 samples, 0.08%) + + + +__perf_event_enable (4 samples, 0.30%) + + + +ip_finish_output (46 samples, 3.50%) +ip_.. + + +__tick_nohz_idle_enter (6 samples, 0.46%) + + + +__skb_clone (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getPrototype (1 samples, 0.08%) + + + +__switch_to (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.write (35 samples, 2.66%) +io.. + + +_raw_spin_lock (1 samples, 0.08%) + + + +_raw_spin_unlock_irqrestore (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.setObjectProp (3 samples, 0.23%) + + + +org/mozilla/javascript/IdScriptableObject:.has (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (2 samples, 0.15%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +idle_cpu (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +io/netty/util/Recycler:.get (1 samples, 0.08%) + + + +native_write_msr_safe (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (2 samples, 0.15%) + + + +ip_output (119 samples, 9.05%) +ip_output + + +io/netty/buffer/AbstractByteBuf:.forEachByteAsc0 (3 samples, 0.23%) + + + +skb_network_protocol (1 samples, 0.08%) + + + +enqueue_entity (5 samples, 0.38%) + + + +tcp_established_options (1 samples, 0.08%) + + + +update_process_times (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.read (3 samples, 0.23%) + + + +update_rq_clock.part.63 (1 samples, 0.08%) + + + +sun/nio/ch/EPollArrayWrapper:.epollWait (4 samples, 0.30%) + + + +sock_poll (2 samples, 0.15%) + + + +io/netty/channel/nio/NioEventLoop:.processSelectedKeysOptimized (949 samples, 72.17%) +io/netty/channel/nio/NioEventLoop:.processSelectedKeysOptimized + + +java_start (985 samples, 74.90%) +java_start + + +mprotect_fixup (1 samples, 0.08%) + + + +ep_scan_ready_list.isra.9 (20 samples, 1.52%) + + + +tcp_v4_do_rcv (23 samples, 1.75%) + + + +sk_stream_alloc_skb (7 samples, 0.53%) + + + +update_curr (2 samples, 0.15%) + + + +tcp_wfree (2 samples, 0.15%) + + + +user_path_at_empty (1 samples, 0.08%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.init (1 samples, 0.08%) + + + +io/netty/buffer/AbstractByteBuf:.writeBytes (1 samples, 0.08%) + + + +sun/nio/ch/NativeThread:.current (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.get (1 samples, 0.08%) + + + +JavaThread::oops_do (3 samples, 0.23%) + + + +org/mozilla/javascript/ScriptableObject:.getBase (2 samples, 0.15%) + + + +ip_local_deliver (1 samples, 0.08%) + + + +org/vertx/java/core/http/impl/ServerConnection:.handleRequest (526 samples, 40.00%) +org/vertx/java/core/http/impl/ServerConnection:.handleRequest + + +__hrtimer_start_range_ns (3 samples, 0.23%) + + + +org/mozilla/javascript/NativeFunction:.initScriptFunction (10 samples, 0.76%) + + + +hrtimer_start (1 samples, 0.08%) + + + +intel_idle (11 samples, 0.84%) + + + +org/mozilla/javascript/IdScriptableObject:.has (3 samples, 0.23%) + + + +ktime_get (1 samples, 0.08%) + + + +update_cfs_rq_blocked_load (1 samples, 0.08%) + + + +org/mozilla/javascript/WrapFactory:.setJavaPrimitiveWrap (1 samples, 0.08%) + + + +sun/nio/ch/NativeThread:.current (1 samples, 0.08%) + + + +system_call_fastpath (28 samples, 2.13%) +s.. + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +sched_clock_cpu (1 samples, 0.08%) + + + +lapic_next_deadline (3 samples, 0.23%) + + + +io/netty/buffer/UnpooledHeapByteBuf:.init (1 samples, 0.08%) + + + +filename_lookup (1 samples, 0.08%) + + + +jni_fast_GetIntField (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.put (6 samples, 0.46%) + + + +x86_pmu_commit_txn (4 samples, 0.30%) + + + +__kfree_skb (1 samples, 0.08%) + + + +org/mozilla/javascript/NativeJavaMethod:.call (10 samples, 0.76%) + + + +process_backlog (34 samples, 2.59%) +pr.. + + +all (1,315 samples, 100%) + + + +io/netty/handler/codec/http/HttpHeaders:.encodeAscii0 (2 samples, 0.15%) + + + +system_call_after_swapgs (6 samples, 0.46%) + + + +_raw_spin_unlock_bh (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +tcp_event_new_data_sent (6 samples, 0.46%) + + + +_raw_spin_unlock_bh (1 samples, 0.08%) + + + +tcp_schedule_loss_probe (3 samples, 0.23%) + + + +tcp_check_space (3 samples, 0.23%) + + + +dev_queue_xmit (4 samples, 0.30%) + + + +tick_nohz_restart (6 samples, 0.46%) + + + +__tcp_ack_snd_check (5 samples, 0.38%) + + + +user_path_at (1 samples, 0.08%) + + + +socket_readable (60 samples, 4.56%) +socke.. + + +org/mozilla/javascript/ScriptableObject:.getSlot (4 samples, 0.30%) + + + +OopMapSet::all_do (1 samples, 0.08%) + + + +socket_writeable (1 samples, 0.08%) + + + +internal_add_timer (1 samples, 0.08%) + + + +select_task_rq_fair (4 samples, 0.30%) + + + +loopback_xmit (5 samples, 0.38%) + + + +sys_epoll_wait (56 samples, 4.26%) +sys_e.. + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +clockevents_program_event (3 samples, 0.23%) + + + +io/netty/buffer/PooledByteBuf:.deallocate (2 samples, 0.15%) + + + +io/netty/util/Recycler:.get (1 samples, 0.08%) + + + +fsnotify (1 samples, 0.08%) + + + +org/mozilla/javascript/NativeJavaMethod:.call (74 samples, 5.63%) +org/moz.. + + +org/mozilla/javascript/ScriptRuntime:.setObjectProp (37 samples, 2.81%) +or.. + + +schedule_preempt_disabled (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.flush (238 samples, 18.10%) +io/netty/channel/AbstractCha.. + + +tcp_queue_rcv (2 samples, 0.15%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (5 samples, 0.38%) + + + +tcp_recvmsg (7 samples, 0.53%) + + + +update_curr (1 samples, 0.08%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +tcp_md5_do_lookup (1 samples, 0.08%) + + + +smp_call_function_single_interrupt (4 samples, 0.30%) + + + +netif_rx (2 samples, 0.15%) + + + +enqueue_to_backlog (1 samples, 0.08%) + + + +_raw_spin_unlock_bh (1 samples, 0.08%) + + + +perf_ioctl (5 samples, 0.38%) + + + +tick_program_event (3 samples, 0.23%) + + + +io/netty/handler/codec/ByteToMessageDecoder:.channelRead (635 samples, 48.29%) +io/netty/handler/codec/ByteToMessageDecoder:.channelRead + + +put_prev_task_fair (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.createSlot (15 samples, 1.14%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +sys_mprotect (1 samples, 0.08%) + + + +Monitor::wait (1 samples, 0.08%) + + + +skb_push (1 samples, 0.08%) + + + +java/lang/ThreadLocal:.get (1 samples, 0.08%) + + + +vtable stub (1 samples, 0.08%) + + + +x86_64_start_kernel (24 samples, 1.83%) +x.. + + +[unknown] (1 samples, 0.08%) + + + +kfree (1 samples, 0.08%) + + + +io/netty/channel/AbstractChannelHandlerContext:.fireChannelReadComplete (241 samples, 18.33%) +io/netty/channel/AbstractCha.. + + +io/netty/buffer/AbstractByteBuf:.forEachByteAsc0 (3 samples, 0.23%) + + + +Java_sun_nio_ch_FileDispatcherImpl_write0 (1 samples, 0.08%) + + + +do_execve_common.isra.22 (1 samples, 0.08%) + + + +group_sched_in (4 samples, 0.30%) + + + +socket_writeable (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject$RelinkedSlot:.getValue (1 samples, 0.08%) + + + +java/lang/String:.hashCode (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getTopLevelScope (1 samples, 0.08%) + + + +kmem_cache_alloc_node (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.has (12 samples, 0.91%) + + + +getnstimeofday (1 samples, 0.08%) + + + +java/util/Arrays:.copyOf (1 samples, 0.08%) + + + +arch_cpu_idle (22 samples, 1.67%) + + + +http_parser_execute (30 samples, 2.28%) +h.. + + +new_slab (2 samples, 0.15%) + + + +io/netty/channel/ChannelDuplexHandler:.flush (1 samples, 0.08%) + + + +generic_smp_call_function_single_interrupt (4 samples, 0.30%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.contains (1 samples, 0.08%) + + + +java/lang/ThreadLocal:.get (1 samples, 0.08%) + + + +security_socket_sendmsg (1 samples, 0.08%) + + + +update_cpu_load_nohz (1 samples, 0.08%) + + + +__perf_event_enable (4 samples, 0.30%) + + + +skb_clone (1 samples, 0.08%) + + + +start_thread (237 samples, 18.02%) +start_thread + + +mod_timer (3 samples, 0.23%) + + + +tcp_data_queue (9 samples, 0.68%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (1 samples, 0.08%) + + + +java/lang/String:.trim (1 samples, 0.08%) + + + +net_rx_action (35 samples, 2.66%) +ne.. + + +inet_sendmsg (177 samples, 13.46%) +inet_sendmsg + + +getnstimeofday (3 samples, 0.23%) + + + +io/netty/buffer/AbstractByteBuf:.writeBytes (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getSlot (3 samples, 0.23%) + + + +[unknown] (1 samples, 0.08%) + + + +hrtimer_try_to_cancel (4 samples, 0.30%) + + + +java/nio/charset/Charset:.lookup (2 samples, 0.15%) + + + +org/mozilla/javascript/NativeJavaObject:.initMembers (1 samples, 0.08%) + + + +start_thread (985 samples, 74.90%) +start_thread + + +x86_pmu_commit_txn (4 samples, 0.30%) + + + +org/mozilla/javascript/ScriptableObject:.getParentScope (1 samples, 0.08%) + + + +system_call_fastpath (5 samples, 0.38%) + + + +_raw_spin_lock (1 samples, 0.08%) + + + +menu_select (4 samples, 0.30%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.add0 (3 samples, 0.23%) + + + +io/netty/buffer/PooledByteBuf:.deallocate (2 samples, 0.15%) + + + +org/mozilla/javascript/BaseFunction:.findInstanceIdInfo (4 samples, 0.30%) + + + +rest_init (24 samples, 1.83%) +r.. + + +ksoftirqd/3 (1 samples, 0.08%) + + + +group_sched_in (4 samples, 0.30%) + + + +account_user_time (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.name (8 samples, 0.61%) + + + +perf_event_for_each_child (5 samples, 0.38%) + + + +io/netty/handler/codec/http/HttpObjectDecoder:.findWhitespace (1 samples, 0.08%) + + + +java/lang/String:.init (1 samples, 0.08%) + + + +set_next_entity (2 samples, 0.15%) + + + +ip_local_deliver_finish (89 samples, 6.77%) +ip_local_.. + + +org/mozilla/javascript/NativeJavaMethod:.findCachedFunction (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.put (12 samples, 0.91%) + + + +hrtimer_start_range_ns (2 samples, 0.15%) + + + +__internal_add_timer (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.getPropFunctionAndThisHelper (9 samples, 0.68%) + + + +smp_call_function_single_interrupt (4 samples, 0.30%) + + + +_raw_spin_lock_bh (1 samples, 0.08%) + + + +__hrtimer_start_range_ns (1 samples, 0.08%) + + + +lock_sock_nested (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject$PrototypeValues:.ensureId (1 samples, 0.08%) + + + +sk_reset_timer (3 samples, 0.23%) + + + +org/mozilla/javascript/gen/file__root_vert_x_2_1_5_sys_mods_io_vertx_lang_js_1_1_0_vertx_http_js_2 (154 samples, 11.71%) +org/mozilla/javas.. + + +io/netty/handler/codec/ByteToMessageDecoder:.channelRead (1 samples, 0.08%) + + + +security_socket_sendmsg (2 samples, 0.15%) + + + +io/netty/handler/codec/http/DefaultHttpHeaders:.add0 (1 samples, 0.08%) + + + +system_call_after_swapgs (1 samples, 0.08%) + + + +hrtimer_cancel (4 samples, 0.30%) + + + +ObjArrayKlass::oop_push_contents (2 samples, 0.15%) + + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (1 samples, 0.08%) + + + +org/mozilla/javascript/IdScriptableObject:.has (1 samples, 0.08%) + + + +schedule_hrtimeout_range (20 samples, 1.52%) + + + +org/mozilla/javascript/ScriptableObject:.putImpl (24 samples, 1.83%) +o.. + + +__fsnotify_parent (1 samples, 0.08%) + + + +vtable stub (1 samples, 0.08%) + + + +__dev_queue_xmit (10 samples, 0.76%) + + + +sys_write (88 samples, 6.69%) +sys_write + + +detach_if_pending (1 samples, 0.08%) + + + +socket_writeable (99 samples, 7.53%) +socket_wri.. + + +org/mozilla/javascript/IdScriptableObject:.findInstanceIdInfo (2 samples, 0.15%) + + + +epoll_ctl (6 samples, 0.46%) + + + +org/vertx/java/core/http/impl/AssembledFullHttpResponse:.toLastContent (1 samples, 0.08%) + + + +org/mozilla/javascript/NativeJavaObject:.get (1 samples, 0.08%) + + + +lock_hrtimer_base.isra.19 (1 samples, 0.08%) + + + +intel_idle (3 samples, 0.23%) + + + +ip_local_deliver (31 samples, 2.36%) +i.. + + +org/mozilla/javascript/IdScriptableObject:.put (23 samples, 1.75%) + + + +io/netty/handler/codec/ByteToMessageDecoder:.channelReadComplete (242 samples, 18.40%) +io/netty/handler/codec/ByteT.. + + +tcp_event_data_recv (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptableObject:.getTopScopeValue (1 samples, 0.08%) + + + +tcp_rearm_rto (3 samples, 0.23%) + + + +org/mozilla/javascript/NativeCall:.init (48 samples, 3.65%) +org/.. + + +tick_nohz_idle_enter (5 samples, 0.38%) + + + +system_call_fastpath (4 samples, 0.30%) + + + +system_call (1 samples, 0.08%) + + + +tick_nohz_idle_enter (6 samples, 0.46%) + + + +tick_program_event (1 samples, 0.08%) + + + +org/mozilla/javascript/ScriptRuntime:.getPropFunctionAndThisHelper (1 samples, 0.08%) + + + +remote_function (4 samples, 0.30%) + + + +tcp_clean_rtx_queue (1 samples, 0.08%) + + + +tick_nohz_idle_exit (7 samples, 0.53%) + + + +org/mozilla/javascript/IdScriptableObject:.put (9 samples, 0.68%) + + + +fsnotify (1 samples, 0.08%) + + + +pick_next_task_fair (2 samples, 0.15%) + + + +do_sync_write (82 samples, 6.24%) +do_sync_.. + + +sys_write (195 samples, 14.83%) +sys_write + + +common_file_perm (1 samples, 0.08%) + + + +account_process_tick (1 samples, 0.08%) + + + diff --git a/benchmarks/cli/_script/flamegraph/files.pl b/benchmarks/cli/_script/flamegraph/files.pl new file mode 100755 index 00000000000..50426b2e47c --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/files.pl @@ -0,0 +1,62 @@ +#!/usr/bin/perl -w +# +# files.pl Print file sizes in folded format, for a flame graph. +# +# This helps you understand storage consumed by a file system, by creating +# a flame graph visualization of space consumed. This is basically a Perl +# version of the "find" command, which emits in folded format for piping +# into flamegraph.pl. +# +# Copyright (c) 2017 Brendan Gregg. +# Licensed under the Apache License, Version 2.0 (the "License") +# +# 03-Feb-2017 Brendan Gregg Created this. + +use strict; +use File::Find; + +sub usage { + print STDERR "USAGE: $0 [--xdev] [DIRECTORY]...\n"; + print STDERR " eg, $0 /Users\n"; + print STDERR " To not descend directories on other filesystems:"; + print STDERR " eg, $0 --xdev /\n"; + print STDERR "Intended to be piped to flamegraph.pl. Full example:\n"; + print STDERR " $0 /Users | flamegraph.pl " . + "--hash --countname=bytes > files.svg\n"; + print STDERR " $0 /usr /home /root /etc | flamegraph.pl " . + "--hash --countname=bytes > files.svg\n"; + print STDERR " $0 --xdev / | flamegraph.pl " . + "--hash --countname=bytes > files.svg\n"; + exit 1; +} + +usage() if @ARGV == 0 or $ARGV[0] eq "--help" or $ARGV[0] eq "-h"; + +my $filter_xdev = 0; +my $xdev_id; + +foreach my $dir (@ARGV) { + if ($dir eq "--xdev") { + $filter_xdev = 1; + } else { + find(\&wanted, $dir); + } +} + +sub wanted { + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size) = lstat($_); + return unless defined $size; + if ($filter_xdev) { + if (!$xdev_id) { + $xdev_id = $dev; + } elsif ($xdev_id ne $dev) { + $File::Find::prune = 1; + return; + } + } + my $path = $File::Find::name; + $path =~ tr/\//;/; # delimiter + $path =~ tr/;.a-zA-Z0-9-/_/c; # ditch whitespace and other chars + $path =~ s/^;//; + print "$path $size\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/flamegraph.pl b/benchmarks/cli/_script/flamegraph/flamegraph.pl new file mode 100755 index 00000000000..4536d98bafc --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/flamegraph.pl @@ -0,0 +1,1318 @@ +#!/usr/bin/perl -w +# +# flamegraph.pl flame stack grapher. +# +# This takes stack samples and renders a call graph, allowing hot functions +# and codepaths to be quickly identified. Stack samples can be generated using +# tools such as DTrace, perf, SystemTap, and Instruments. +# +# USAGE: ./flamegraph.pl [options] input.txt > graph.svg +# +# grep funcA input.txt | ./flamegraph.pl [options] > graph.svg +# +# Then open the resulting .svg in a web browser, for interactivity: mouse-over +# frames for info, click to zoom, and ctrl-F to search. +# +# Options are listed in the usage message (--help). +# +# The input is stack frames and sample counts formatted as single lines. Each +# frame in the stack is semicolon separated, with a space and count at the end +# of the line. These can be generated for Linux perf script output using +# stackcollapse-perf.pl, for DTrace using stackcollapse.pl, and for other tools +# using the other stackcollapse programs. Example input: +# +# swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 1 +# +# An optional extra column of counts can be provided to generate a differential +# flame graph of the counts, colored red for more, and blue for less. This +# can be useful when using flame graphs for non-regression testing. +# See the header comment in the difffolded.pl program for instructions. +# +# The input functions can optionally have annotations at the end of each +# function name, following a precedent by some tools (Linux perf's _[k]): +# _[k] for kernel +# _[i] for inlined +# _[j] for jit +# _[w] for waker +# Some of the stackcollapse programs support adding these annotations, eg, +# stackcollapse-perf.pl --kernel --jit. They are used merely for colors by +# some palettes, eg, flamegraph.pl --color=java. +# +# The output flame graph shows relative presence of functions in stack samples. +# The ordering on the x-axis has no meaning; since the data is samples, time +# order of events is not known. The order used sorts function names +# alphabetically. +# +# While intended to process stack samples, this can also process stack traces. +# For example, tracing stacks for memory allocation, or resource usage. You +# can use --title to set the title to reflect the content, and --countname +# to change "samples" to "bytes" etc. +# +# There are a few different palettes, selectable using --color. By default, +# the colors are selected at random (except for differentials). Functions +# called "-" will be printed gray, which can be used for stack separators (eg, +# between user and kernel stacks). +# +# HISTORY +# +# This was inspired by Neelakanth Nadgir's excellent function_call_graph.rb +# program, which visualized function entry and return trace events. As Neel +# wrote: "The output displayed is inspired by Roch's CallStackAnalyzer which +# was in turn inspired by the work on vftrace by Jan Boerhout". See: +# https://blogs.oracle.com/realneel/entry/visualizing_callstacks_via_dtrace_and +# +# Copyright 2016 Netflix, Inc. +# Copyright 2011 Joyent, Inc. All rights reserved. +# Copyright 2011 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 11-Oct-2014 Adrien Mahieux Added zoom. +# 21-Nov-2013 Shawn Sterling Added consistent palette file option +# 17-Mar-2013 Tim Bunce Added options and more tunables. +# 15-Dec-2011 Dave Pacheco Support for frames with whitespace. +# 10-Sep-2011 Brendan Gregg Created this. + +use strict; + +use Getopt::Long; + +use open qw(:std :utf8); + +# tunables +my $encoding; +my $fonttype = "Verdana"; +my $imagewidth = 1200; # max width, pixels +my $frameheight = 16; # max height is dynamic +my $fontsize = 12; # base text size +my $fontwidth = 0.59; # avg width relative to fontsize +my $minwidth = 0.1; # min function width, pixels or percentage of time +my $nametype = "Function:"; # what are the names in the data? +my $countname = "samples"; # what are the counts in the data? +my $colors = "hot"; # color theme +my $bgcolors = ""; # background color theme +my $nameattrfile; # file holding function attributes +my $timemax; # (override the) sum of the counts +my $factor = 1; # factor to scale counts by +my $hash = 0; # color by function name +my $rand = 0; # color randomly +my $palette = 0; # if we use consistent palettes (default off) +my %palette_map; # palette map hash +my $pal_file = "palette.map"; # palette map file name +my $stackreverse = 0; # reverse stack order, switching merge end +my $inverted = 0; # icicle graph +my $flamechart = 0; # produce a flame chart (sort by time, do not merge stacks) +my $negate = 0; # switch differential hues +my $titletext = "\000"; # centered heading +my $titledefault = "Flame Graph"; # overwritten by --title +my $titleinverted = "Icicle Graph"; # " " +my $searchcolor = "rgb(230,0,230)"; # color for search highlighting +my $notestext = ""; # embedded notes in SVG +my $subtitletext = ""; # second level title (optional) +my $help = 0; + +sub usage { + die < outfile.svg\n + --title TEXT # change title text + --subtitle TEXT # second level title (optional) + --width NUM # width of image (default 1200) + --height NUM # height of each frame (default 16) + --minwidth NUM # omit smaller functions. In pixels or use "%" for + # percentage of time (default 0.1 pixels) + --fonttype FONT # font type (default "Verdana") + --fontsize NUM # font size (default 12) + --countname TEXT # count type label (default "samples") + --nametype TEXT # name type label (default "Function:") + --colors PALETTE # set color palette. choices are: hot (default), mem, + # io, wakeup, chain, java, js, perl, red, green, blue, + # aqua, yellow, purple, orange + --bgcolors COLOR # set background colors. gradient choices are yellow + # (default), blue, green, grey; flat colors use "#rrggbb"; + # or none to omit a background + --hash # colors are keyed by function name hash + --random # colors are randomly generated + --cp # use consistent palette (palette.map) + --reverse # generate stack-reversed flame graph + --inverted # icicle graph + --flamechart # produce a flame chart (sort by time, do not merge stacks) + --negate # switch differential hues (blue<->red) + --notes TEXT # add notes comment in SVG (for debugging) + --help # this message + + eg, + $0 --title="Flame Graph: malloc()" trace.txt > graph.svg +USAGE_END +} + +GetOptions( + 'fonttype=s' => \$fonttype, + 'width=i' => \$imagewidth, + 'height=i' => \$frameheight, + 'encoding=s' => \$encoding, + 'fontsize=f' => \$fontsize, + 'fontwidth=f' => \$fontwidth, + 'minwidth=s' => \$minwidth, + 'title=s' => \$titletext, + 'subtitle=s' => \$subtitletext, + 'nametype=s' => \$nametype, + 'countname=s' => \$countname, + 'nameattr=s' => \$nameattrfile, + 'total=s' => \$timemax, + 'factor=f' => \$factor, + 'colors=s' => \$colors, + 'bgcolors=s' => \$bgcolors, + 'hash' => \$hash, + 'random' => \$rand, + 'cp' => \$palette, + 'reverse' => \$stackreverse, + 'inverted' => \$inverted, + 'flamechart' => \$flamechart, + 'negate' => \$negate, + 'notes=s' => \$notestext, + 'help' => \$help, +) or usage(); +$help && usage(); + +# internals +my $ypad1 = $fontsize * 3; # pad top, include title +my $ypad2 = $fontsize * 2 + 10; # pad bottom, include labels +my $ypad3 = $fontsize * 2; # pad top, include subtitle (optional) +my $xpad = 10; # pad lefm and right +my $framepad = 1; # vertical padding for frames +my $depthmax = 0; +my %Events; +my %nameattr; + +if ($flamechart && $titletext eq "\000") { + $titletext = "Flame Chart"; +} + +if ($titletext eq "\000") { + unless ($inverted) { + $titletext = $titledefault; + } else { + $titletext = $titleinverted; + } +} + +if ($nameattrfile) { + # The name-attribute file format is a function name followed by a tab then + # a sequence of tab separated name=value pairs. + open my $attrfh, $nameattrfile or die "Can't read $nameattrfile: $!\n"; + while (<$attrfh>) { + chomp; + my ($funcname, $attrstr) = split /\t/, $_, 2; + die "Invalid format in $nameattrfile" unless defined $attrstr; + $nameattr{$funcname} = { map { split /=/, $_, 2 } split /\t/, $attrstr }; + } +} + +if ($notestext =~ /[<>]/) { + die "Notes string can't contain < or >" +} + +# Ensure minwidth is a valid floating-point number, +# print usage string if not +my $minwidth_f; +if ($minwidth =~ /^([0-9.]+)%?$/) { + $minwidth_f = $1; +} else { + warn "Value '$minwidth' is invalid for minwidth, expected a float.\n"; + usage(); +} + +# background colors: +# - yellow gradient: default (hot, java, js, perl) +# - green gradient: mem +# - blue gradient: io, wakeup, chain +# - gray gradient: flat colors (red, green, blue, ...) +if ($bgcolors eq "") { + # choose a default + if ($colors eq "mem") { + $bgcolors = "green"; + } elsif ($colors =~ /^(io|wakeup|chain)$/) { + $bgcolors = "blue"; + } elsif ($colors =~ /^(red|green|blue|aqua|yellow|purple|orange)$/) { + $bgcolors = "grey"; + } else { + $bgcolors = "yellow"; + } +} +my ($bgcolor1, $bgcolor2); +if ($bgcolors eq "yellow") { + $bgcolor1 = "#eeeeee"; # background color gradient start + $bgcolor2 = "#eeeeb0"; # background color gradient stop +} elsif ($bgcolors eq "blue") { + $bgcolor1 = "#eeeeee"; $bgcolor2 = "#e0e0ff"; +} elsif ($bgcolors eq "green") { + $bgcolor1 = "#eef2ee"; $bgcolor2 = "#e0ffe0"; +} elsif ($bgcolors eq "grey") { + $bgcolor1 = "#f8f8f8"; $bgcolor2 = "#e8e8e8"; +} elsif ($bgcolors =~ /^#......$/) { + $bgcolor1 = $bgcolor2 = $bgcolors; +} elsif ($bgcolors ne 'none') { + die "Unrecognized bgcolor option \"$bgcolors\"" +} + +# SVG functions +{ package SVG; + sub new { + my $class = shift; + my $self = {}; + bless ($self, $class); + return $self; + } + + sub header { + my ($self, $w, $h) = @_; + my $enc_attr = ''; + if (defined $encoding) { + $enc_attr = qq{ encoding="$encoding"}; + } + $self->{svg} .= < + + + + +SVG + } + + sub include { + my ($self, $content) = @_; + $self->{svg} .= $content; + } + + sub colorAllocate { + my ($self, $r, $g, $b) = @_; + return "rgb($r,$g,$b)"; + } + + sub group_start { + my ($self, $attr) = @_; + + my @g_attr = map { + exists $attr->{$_} ? sprintf(qq/$_="%s"/, $attr->{$_}) : () + } qw(id class); + push @g_attr, $attr->{g_extra} if $attr->{g_extra}; + if ($attr->{href}) { + my @a_attr; + push @a_attr, sprintf qq/xlink:href="%s"/, $attr->{href} if $attr->{href}; + # default target=_top else links will open within SVG + push @a_attr, sprintf qq/target="%s"/, $attr->{target} || "_top"; + push @a_attr, $attr->{a_extra} if $attr->{a_extra}; + $self->{svg} .= sprintf qq/\n/, join(' ', (@a_attr, @g_attr)); + } else { + $self->{svg} .= sprintf qq/\n/, join(' ', @g_attr); + } + + $self->{svg} .= sprintf qq/%s<\/title>/, $attr->{title} + if $attr->{title}; # should be first element within g container + } + + sub group_end { + my ($self, $attr) = @_; + $self->{svg} .= $attr->{href} ? qq/<\/a>\n/ : qq/<\/g>\n/; + } + + sub filledRectangle { + my ($self, $x1, $y1, $x2, $y2, $fill, $extra) = @_; + $x1 = sprintf "%0.1f", $x1; + $x2 = sprintf "%0.1f", $x2; + my $w = sprintf "%0.1f", $x2 - $x1; + my $h = sprintf "%0.1f", $y2 - $y1; + $extra = defined $extra ? $extra : ""; + $self->{svg} .= qq/\n/; + } + + sub stringTTF { + my ($self, $id, $x, $y, $str, $extra) = @_; + $x = sprintf "%0.2f", $x; + $id = defined $id ? qq/id="$id"/ : ""; + $extra ||= ""; + $self->{svg} .= qq/$str<\/text>\n/; + } + + sub svg { + my $self = shift; + return "$self->{svg}\n"; + } + 1; +} + +sub namehash { + # Generate a vector hash for the name string, weighting early over + # later characters. We want to pick the same colors for function + # names across different flame graphs. + my $name = shift; + my $vector = 0; + my $weight = 1; + my $max = 1; + my $mod = 10; + # if module name present, trunc to 1st char + $name =~ s/.(.*?)`//; + foreach my $c (split //, $name) { + my $i = (ord $c) % $mod; + $vector += ($i / ($mod++ - 1)) * $weight; + $max += 1 * $weight; + $weight *= 0.70; + last if $mod > 12; + } + return (1 - $vector / $max) +} + +sub sum_namehash { + my $name = shift; + return unpack("%32W*", $name); +} + +sub random_namehash { + # Generate a random hash for the name string. + # This ensures that functions with the same name have the same color, + # both within a flamegraph and across multiple flamegraphs without + # needing to set a palette and while preserving the original flamegraph + # optic, unlike what happens with --hash. + my $name = shift; + my $hash = sum_namehash($name); + srand($hash); + return rand(1) +} + +sub color { + my ($type, $hash, $name) = @_; + my ($v1, $v2, $v3); + + if ($hash) { + $v1 = namehash($name); + $v2 = $v3 = namehash(scalar reverse $name); + } elsif ($rand) { + $v1 = rand(1); + $v2 = rand(1); + $v3 = rand(1); + } else { + $v1 = random_namehash($name); + $v2 = random_namehash($name); + $v3 = random_namehash($name); + } + + # theme palettes + if (defined $type and $type eq "hot") { + my $r = 205 + int(50 * $v3); + my $g = 0 + int(230 * $v1); + my $b = 0 + int(55 * $v2); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "mem") { + my $r = 0; + my $g = 190 + int(50 * $v2); + my $b = 0 + int(210 * $v1); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "io") { + my $r = 80 + int(60 * $v1); + my $g = $r; + my $b = 190 + int(55 * $v2); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "libgit2") { + my $alpha = sprintf("%.2f", 0.2 + (0.8 * (($v1 + $v2) / 2))); + return ($v3 > 0.5) ? + "rgba(241,80,47,$alpha)" : + "rgba(55,125,205,$alpha)"; + } + + # multi palettes + if (defined $type and $type eq "java") { + # Handle both annotations (_[j], _[i], ...; which are + # accurate), as well as input that lacks any annotations, as + # best as possible. Without annotations, we get a little hacky + # and match on java|org|com, etc. + if ($name =~ m:_\[j\]$:) { # jit annotation + $type = "green"; + } elsif ($name =~ m:_\[i\]$:) { # inline annotation + $type = "aqua"; + } elsif ($name =~ m:^L?(java|javax|jdk|net|org|com|io|sun)/:) { # Java + $type = "green"; + } elsif ($name =~ /:::/) { # Java, typical perf-map-agent method separator + $type = "green"; + } elsif ($name =~ /::/) { # C++ + $type = "yellow"; + } elsif ($name =~ m:_\[k\]$:) { # kernel annotation + $type = "orange"; + } elsif ($name =~ /::/) { # C++ + $type = "yellow"; + } else { # system + $type = "red"; + } + # fall-through to color palettes + } + if (defined $type and $type eq "perl") { + if ($name =~ /::/) { # C++ + $type = "yellow"; + } elsif ($name =~ m:Perl: or $name =~ m:\.pl:) { # Perl + $type = "green"; + } elsif ($name =~ m:_\[k\]$:) { # kernel + $type = "orange"; + } else { # system + $type = "red"; + } + # fall-through to color palettes + } + if (defined $type and $type eq "js") { + # Handle both annotations (_[j], _[i], ...; which are + # accurate), as well as input that lacks any annotations, as + # best as possible. Without annotations, we get a little hacky, + # and match on a "/" with a ".js", etc. + if ($name =~ m:_\[j\]$:) { # jit annotation + if ($name =~ m:/:) { + $type = "green"; # source + } else { + $type = "aqua"; # builtin + } + } elsif ($name =~ /::/) { # C++ + $type = "yellow"; + } elsif ($name =~ m:/.*\.js:) { # JavaScript (match "/" in path) + $type = "green"; + } elsif ($name =~ m/:/) { # JavaScript (match ":" in builtin) + $type = "aqua"; + } elsif ($name =~ m/^ $/) { # Missing symbol + $type = "green"; + } elsif ($name =~ m:_\[k\]:) { # kernel + $type = "orange"; + } else { # system + $type = "red"; + } + # fall-through to color palettes + } + if (defined $type and $type eq "wakeup") { + $type = "aqua"; + # fall-through to color palettes + } + if (defined $type and $type eq "chain") { + if ($name =~ m:_\[w\]:) { # waker + $type = "aqua" + } else { # off-CPU + $type = "blue"; + } + # fall-through to color palettes + } + + # color palettes + if (defined $type and $type eq "red") { + my $r = 200 + int(55 * $v1); + my $x = 50 + int(80 * $v1); + return "rgb($r,$x,$x)"; + } + if (defined $type and $type eq "green") { + my $g = 200 + int(55 * $v1); + my $x = 50 + int(60 * $v1); + return "rgb($x,$g,$x)"; + } + if (defined $type and $type eq "blue") { + my $b = 205 + int(50 * $v1); + my $x = 80 + int(60 * $v1); + return "rgb($x,$x,$b)"; + } + if (defined $type and $type eq "yellow") { + my $x = 175 + int(55 * $v1); + my $b = 50 + int(20 * $v1); + return "rgb($x,$x,$b)"; + } + if (defined $type and $type eq "purple") { + my $x = 190 + int(65 * $v1); + my $g = 80 + int(60 * $v1); + return "rgb($x,$g,$x)"; + } + if (defined $type and $type eq "aqua") { + my $r = 50 + int(60 * $v1); + my $g = 165 + int(55 * $v1); + my $b = 165 + int(55 * $v1); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "orange") { + my $r = 190 + int(65 * $v1); + my $g = 90 + int(65 * $v1); + return "rgb($r,$g,0)"; + } + + return "rgb(0,0,0)"; +} + +sub color_scale { + my ($value, $max) = @_; + my ($r, $g, $b) = (255, 255, 255); + $value = -$value if $negate; + if ($value > 0) { + $g = $b = int(210 * ($max - $value) / $max); + } elsif ($value < 0) { + $r = $g = int(210 * ($max + $value) / $max); + } + return "rgb($r,$g,$b)"; +} + +sub color_map { + my ($colors, $func) = @_; + if (exists $palette_map{$func}) { + return $palette_map{$func}; + } else { + $palette_map{$func} = color($colors, $hash, $func); + return $palette_map{$func}; + } +} + +sub write_palette { + open(FILE, ">$pal_file"); + foreach my $key (sort keys %palette_map) { + print FILE $key."->".$palette_map{$key}."\n"; + } + close(FILE); +} + +sub read_palette { + if (-e $pal_file) { + open(FILE, $pal_file) or die "can't open file $pal_file: $!"; + while ( my $line = ) { + chomp($line); + (my $key, my $value) = split("->",$line); + $palette_map{$key}=$value; + } + close(FILE) + } +} + +my %Node; # Hash of merged frame data +my %Tmp; + +# flow() merges two stacks, storing the merged frames and value data in %Node. +sub flow { + my ($last, $this, $v, $d) = @_; + + my $len_a = @$last - 1; + my $len_b = @$this - 1; + + my $i = 0; + my $len_same; + for (; $i <= $len_a; $i++) { + last if $i > $len_b; + last if $last->[$i] ne $this->[$i]; + } + $len_same = $i; + + for ($i = $len_a; $i >= $len_same; $i--) { + my $k = "$last->[$i];$i"; + # a unique ID is constructed from "func;depth;etime"; + # func-depth isn't unique, it may be repeated later. + $Node{"$k;$v"}->{stime} = delete $Tmp{$k}->{stime}; + if (defined $Tmp{$k}->{delta}) { + $Node{"$k;$v"}->{delta} = delete $Tmp{$k}->{delta}; + } + delete $Tmp{$k}; + } + + for ($i = $len_same; $i <= $len_b; $i++) { + my $k = "$this->[$i];$i"; + $Tmp{$k}->{stime} = $v; + if (defined $d) { + $Tmp{$k}->{delta} += $i == $len_b ? $d : 0; + } + } + + return $this; +} + +# parse input +my @Data; +my @SortedData; +my $last = []; +my $time = 0; +my $delta = undef; +my $ignored = 0; +my $line; +my $maxdelta = 1; + +# reverse if needed +foreach (<>) { + chomp; + $line = $_; + if ($stackreverse) { + # there may be an extra samples column for differentials + # XXX todo: redo these REs as one. It's repeated below. + my($stack, $samples) = (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + my $samples2 = undef; + if ($stack =~ /^(.*)\s+?(\d+(?:\.\d*)?)$/) { + $samples2 = $samples; + ($stack, $samples) = $stack =~ (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + unshift @Data, join(";", reverse split(";", $stack)) . " $samples $samples2"; + } else { + unshift @Data, join(";", reverse split(";", $stack)) . " $samples"; + } + } else { + unshift @Data, $line; + } +} + +if ($flamechart) { + # In flame chart mode, just reverse the data so time moves from left to right. + @SortedData = reverse @Data; +} else { + @SortedData = sort @Data; +} + +# process and merge frames +foreach (@SortedData) { + chomp; + # process: folded_stack count + # eg: func_a;func_b;func_c 31 + my ($stack, $samples) = (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + unless (defined $samples and defined $stack) { + ++$ignored; + next; + } + + # there may be an extra samples column for differentials: + my $samples2 = undef; + if ($stack =~ /^(.*)\s+?(\d+(?:\.\d*)?)$/) { + $samples2 = $samples; + ($stack, $samples) = $stack =~ (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + } + $delta = undef; + if (defined $samples2) { + $delta = $samples2 - $samples; + $maxdelta = abs($delta) if abs($delta) > $maxdelta; + } + + # for chain graphs, annotate waker frames with "_[w]", for later + # coloring. This is a hack, but has a precedent ("_[k]" from perf). + if ($colors eq "chain") { + my @parts = split ";--;", $stack; + my @newparts = (); + $stack = shift @parts; + $stack .= ";--;"; + foreach my $part (@parts) { + $part =~ s/;/_[w];/g; + $part .= "_[w]"; + push @newparts, $part; + } + $stack .= join ";--;", @parts; + } + + # merge frames and populate %Node: + $last = flow($last, [ '', split ";", $stack ], $time, $delta); + + if (defined $samples2) { + $time += $samples2; + } else { + $time += $samples; + } +} +flow($last, [], $time, $delta); + +if ($countname eq "samples") { + # If $countname is used, it's likely that we're not measuring in stack samples + # (e.g. time could be the unit), so don't warn. + warn "Stack count is low ($time). Did something go wrong?\n" if $time < 100; +} + +warn "Ignored $ignored lines with invalid format\n" if $ignored; +unless ($time) { + warn "ERROR: No stack counts found\n"; + my $im = SVG->new(); + # emit an error message SVG, for tools automating flamegraph use + my $imageheight = $fontsize * 5; + $im->header($imagewidth, $imageheight); + $im->stringTTF(undef, int($imagewidth / 2), $fontsize * 2, + "ERROR: No valid input provided to flamegraph.pl."); + print $im->svg; + exit 2; +} +if ($timemax and $timemax < $time) { + warn "Specified --total $timemax is less than actual total $time, so ignored\n" + if $timemax/$time > 0.02; # only warn is significant (e.g., not rounding etc) + undef $timemax; +} +$timemax ||= $time; + +my $widthpertime = ($imagewidth - 2 * $xpad) / $timemax; + +# Treat as a percentage of time if the string ends in a "%". +my $minwidth_time; +if ($minwidth =~ /%$/) { + $minwidth_time = $timemax * $minwidth_f / 100; +} else { + $minwidth_time = $minwidth_f / $widthpertime; +} + +# prune blocks that are too narrow and determine max depth +while (my ($id, $node) = each %Node) { + my ($func, $depth, $etime) = split ";", $id; + my $stime = $node->{stime}; + die "missing start for $id" if not defined $stime; + + if (($etime-$stime) < $minwidth_time) { + delete $Node{$id}; + next; + } + $depthmax = $depth if $depth > $depthmax; +} + +# draw canvas, and embed interactive JavaScript program +my $imageheight = (($depthmax + 1) * $frameheight) + $ypad1 + $ypad2; +$imageheight += $ypad3 if $subtitletext ne ""; +my $titlesize = $fontsize + 5; +my $im = SVG->new(); +my ($black, $vdgrey, $dgrey) = ( + $im->colorAllocate(0, 0, 0), + $im->colorAllocate(160, 160, 160), + $im->colorAllocate(200, 200, 200), + ); +$im->header($imagewidth, $imageheight); +my $backgroundinc = ''; + +if ($bgcolors ne 'none') { + $backgroundinc = < + + + + + +INC +} + +my $inc = < + text { font-family:$fonttype; font-size:${fontsize}px; fill:$black; } + #search, #ignorecase { opacity:0.1; cursor:pointer; } + #search:hover, #search.show, #ignorecase:hover, #ignorecase.show { opacity:1; } + #subtitle { text-anchor:middle; font-color:$vdgrey; } + #title { text-anchor:middle; font-size:${titlesize}px} + #unzoom { cursor:pointer; } + #frames > *:hover { stroke:black; stroke-width:0.5; cursor:pointer; } + .hide { display:none; } + .parent { opacity:0.5; } + + +INC +$im->include($inc); +$im->filledRectangle(0, 0, $imagewidth, $imageheight, 'url(#background)'); +$im->stringTTF("title", int($imagewidth / 2), $fontsize * 2, $titletext) if $titletext ne ""; +$im->stringTTF("subtitle", int($imagewidth / 2), $fontsize * 4, $subtitletext) if $subtitletext ne ""; +$im->stringTTF("details", $xpad, $imageheight - ($ypad2 / 2), " "); +$im->stringTTF("unzoom", $xpad, $fontsize * 2, "Reset Zoom", 'class="hide"'); +$im->stringTTF("search", $imagewidth - $xpad - 100, $fontsize * 2, "Search"); +$im->stringTTF("ignorecase", $imagewidth - $xpad - 16, $fontsize * 2, "ic"); +$im->stringTTF("matched", $imagewidth - $xpad - 100, $imageheight - ($ypad2 / 2), " "); + +if ($palette) { + read_palette(); +} + +# draw frames +$im->group_start({id => "frames"}); +while (my ($id, $node) = each %Node) { + my ($func, $depth, $etime) = split ";", $id; + my $stime = $node->{stime}; + my $delta = $node->{delta}; + + $etime = $timemax if $func eq "" and $depth == 0; + + my $x1 = $xpad + $stime * $widthpertime; + my $x2 = $xpad + $etime * $widthpertime; + my ($y1, $y2); + unless ($inverted) { + $y1 = $imageheight - $ypad2 - ($depth + 1) * $frameheight + $framepad; + $y2 = $imageheight - $ypad2 - $depth * $frameheight; + } else { + $y1 = $ypad1 + $depth * $frameheight; + $y2 = $ypad1 + ($depth + 1) * $frameheight - $framepad; + } + + # Add commas per perlfaq5: + # https://perldoc.perl.org/perlfaq5#How-can-I-output-my-numbers-with-commas-added? + my $samples = sprintf "%.0f", ($etime - $stime) * $factor; + (my $samples_txt = $samples) + =~ s/(^[-+]?\d+?(?=(?>(?:\d{3})+)(?!\d))|\G\d{3}(?=\d))/$1,/g; + + my $info; + if ($func eq "" and $depth == 0) { + $info = "all ($samples_txt $countname, 100%)"; + } else { + my $pct = sprintf "%.2f", ((100 * $samples) / ($timemax * $factor)); + my $escaped_func = $func; + # clean up SVG breaking characters: + $escaped_func =~ s/&/&/g; + $escaped_func =~ s//>/g; + $escaped_func =~ s/"/"/g; + $escaped_func =~ s/_\[[kwij]\]$//; # strip any annotation + unless (defined $delta) { + $info = "$escaped_func ($samples_txt $countname, $pct%)"; + } else { + my $d = $negate ? -$delta : $delta; + my $deltapct = sprintf "%.2f", ((100 * $d) / ($timemax * $factor)); + $deltapct = $d > 0 ? "+$deltapct" : $deltapct; + $info = "$escaped_func ($samples_txt $countname, $pct%; $deltapct%)"; + } + } + + my $nameattr = { %{ $nameattr{$func}||{} } }; # shallow clone + $nameattr->{title} ||= $info; + $im->group_start($nameattr); + + my $color; + if ($func eq "--") { + $color = $vdgrey; + } elsif ($func eq "-") { + $color = $dgrey; + } elsif (defined $delta) { + $color = color_scale($delta, $maxdelta); + } elsif ($palette) { + $color = color_map($colors, $func); + } else { + $color = color($colors, $hash, $func); + } + $im->filledRectangle($x1, $y1, $x2, $y2, $color, 'rx="2" ry="2"'); + + my $chars = int( ($x2 - $x1) / ($fontsize * $fontwidth)); + my $text = ""; + if ($chars >= 3) { # room for one char plus two dots + $func =~ s/_\[[kwij]\]$//; # strip any annotation + $text = substr $func, 0, $chars; + substr($text, -2, 2) = ".." if $chars < length $func; + $text =~ s/&/&/g; + $text =~ s//>/g; + } + $im->stringTTF(undef, $x1 + 3, 3 + ($y1 + $y2) / 2, $text); + + $im->group_end($nameattr); +} +$im->group_end(); + +print $im->svg; + +if ($palette) { + write_palette(); +} + +# vim: ts=8 sts=8 sw=8 noexpandtab diff --git a/benchmarks/cli/_script/flamegraph/jmaps b/benchmarks/cli/_script/flamegraph/jmaps new file mode 100755 index 00000000000..f8014f5a82f --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/jmaps @@ -0,0 +1,104 @@ +#!/bin/bash +# +# jmaps - creates java /tmp/perf-PID.map symbol maps for all java processes. +# +# This is a helper script that finds all running "java" processes, then executes +# perf-map-agent on them all, creating symbol map files in /tmp. These map files +# are read by perf_events (aka "perf") when doing system profiles (specifically, +# the "report" and "script" subcommands). +# +# USAGE: jmaps [-u] +# -u # unfoldall: include inlined symbols +# +# My typical workflow is this: +# +# perf record -F 99 -a -g -- sleep 30; jmaps +# perf script > out.stacks +# ./stackcollapse-perf.pl out.stacks | ./flamegraph.pl --color=java --hash > out.stacks.svg +# +# The stackcollapse-perf.pl and flamegraph.pl programs come from: +# https://github.com/brendangregg/FlameGraph +# +# REQUIREMENTS: +# Tune two environment settings below. +# +# 13-Feb-2015 Brendan Gregg Created this. +# 20-Feb-2017 " " Added -u for unfoldall. + +JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-8-oracle} +AGENT_HOME=${AGENT_HOME:-/usr/lib/jvm/perf-map-agent} # from https://github.com/jvm-profiling-tools/perf-map-agent +debug=0 + +if [[ "$USER" != root ]]; then + echo "ERROR: not root user? exiting..." + exit +fi + +if [[ ! -x $JAVA_HOME ]]; then + echo "ERROR: JAVA_HOME not set correctly; edit $0 and fix" + exit +fi + +if [[ ! -x $AGENT_HOME ]]; then + echo "ERROR: AGENT_HOME not set correctly; edit $0 and fix" + exit +fi + +if [[ "$1" == "-u" ]]; then + opts=unfoldall +fi + +# figure out where the agent files are: +AGENT_OUT="" +AGENT_JAR="" +if [[ -e $AGENT_HOME/out/attach-main.jar ]]; then + AGENT_JAR=$AGENT_HOME/out/attach-main.jar +elif [[ -e $AGENT_HOME/attach-main.jar ]]; then + AGENT_JAR=$AGENT_HOME/attach-main.jar +fi +if [[ -e $AGENT_HOME/out/libperfmap.so ]]; then + AGENT_OUT=$AGENT_HOME/out +elif [[ -e $AGENT_HOME/libperfmap.so ]]; then + AGENT_OUT=$AGENT_HOME +fi +if [[ "$AGENT_OUT" == "" || "$AGENT_JAR" == "" ]]; then + echo "ERROR: Missing perf-map-agent files in $AGENT_HOME. Check installation." + exit +fi + +# Fetch map for all "java" processes +echo "Fetching maps for all java processes..." +for pid in $(pgrep -x java); do + mapfile=/tmp/perf-$pid.map + [[ -e $mapfile ]] && rm $mapfile + + cmd="cd $AGENT_OUT; $JAVA_HOME/bin/java -Xms32m -Xmx128m -cp $AGENT_JAR:$JAVA_HOME/lib/tools.jar net.virtualvoid.perf.AttachOnce $pid $opts" + (( debug )) && echo $cmd + + user=$(ps ho user -p $pid) + group=$(ps ho group -p $pid) + if [[ "$user" != root ]]; then + if [[ "$user" == [0-9]* ]]; then + # UID only, likely GID too, run sudo with #UID: + cmd="sudo -u '#'$user -g '#'$group sh -c '$cmd'" + else + cmd="sudo -u $user -g $group sh -c '$cmd'" + fi + fi + + echo "Mapping PID $pid (user $user):" + if (( debug )); then + time eval $cmd + else + eval $cmd + fi + if [[ -e "$mapfile" ]]; then + chown root $mapfile + chmod 666 $mapfile + else + echo "ERROR: $mapfile not created." + fi + + echo "wc(1): $(wc $mapfile)" + echo +done diff --git a/benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl b/benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl new file mode 100755 index 00000000000..3a9902da49f --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl -w +# +# pkgsplit-perf.pl Split IP samples on package names "/", eg, Java. +# +# This is for the creation of Java package flame graphs. Example steps: +# +# perf record -F 199 -a -- sleep 30; ./jmaps +# perf script | ./pkgsplit-perf.pl | ./flamegraph.pl > out.svg +# +# Note that stack traces are not sampled (no -g), as we split Java package +# names into frames rather than stack frames. +# +# (jmaps is a helper script for automating perf-map-agent: Java symbol dumps.) +# +# The default output of "perf script" varies between kernel versions, so we'll +# need to deal with that here. I could make people use the perf script option +# to pick fields, so our input is static, but A) I prefer the simplicity of +# just saying: run "perf script", and B) the option to choose fields itself +# changed between kernel versions! -f became -F. +# +# Copyright 2017 Netflix, Inc. +# Licensed under the Apache License, Version 2.0 (the "License") +# +# 20-Sep-2016 Brendan Gregg Created this. + +use strict; + +my $include_pname = 1; # include process names in stacks +my $include_pid = 0; # include process ID with process name +my $include_tid = 0; # include process & thread ID with process name + +while (<>) { + # filter comments + next if /^#/; + + # filter idle events + next if /xen_hypercall_sched_op|cpu_idle|native_safe_halt/; + + my ($pid, $tid, $pname); + + # Linux 3.13: + # java 13905 [000] 8048.096572: cpu-clock: 7fd781ac3053 Ljava/util/Arrays$ArrayList;::toArray (/tmp/perf-12149.map) + # java 8301 [050] 13527.392454: cycles: 7fa8a80d9bff Dictionary::find(int, unsigned int, Symbol*, ClassLoaderData*, Handle, Thread*) (/usr/lib/jvm/java-8-oracle-1.8.0.121/jre/lib/amd64/server/libjvm.so) + # java 4567/8603 [023] 13527.389886: cycles: 7fa863349895 Lcom/google/gson/JsonObject;::add (/tmp/perf-4567.map) + # + # Linux 4.8: + # java 30894 [007] 452884.077440: 10101010 cpu-clock: 7f0acc8eff67 Lsun/nio/ch/SocketChannelImpl;::read+0x27 (/tmp/perf-30849.map) + # bash 26858/26858 [006] 5440237.995639: cpu-clock: 433573 [unknown] (/bin/bash) + # + if (/^\s+(\S.+?)\s+(\d+)\/*(\d+)*\s.*?:.*:/) { + # parse process name and pid/tid + if ($3) { + ($pid, $tid) = ($2, $3); + } else { + ($pid, $tid) = ("?", $2); + } + + if ($include_tid) { + $pname = "$1-$pid/$tid"; + } elsif ($include_pid) { + $pname = "$1-$pid"; + } else { + $pname = $1; + } + $pname =~ tr/ /_/; + } else { + # not a match + next; + } + + # parse rest of line + s/^.*?:.*?:\s+//; + s/ \(.*?\)$//; + chomp; + my ($addr, $func) = split(' ', $_, 2); + + # strip Java's leading "L" + $func =~ s/^L//; + + # replace numbers with X + $func =~ s/[0-9]/X/g; + + # colon delimitered + $func =~ s:/:;:g; + print "$pname;$func 1\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/range-perf.pl b/benchmarks/cli/_script/flamegraph/range-perf.pl new file mode 100755 index 00000000000..0fca6decd23 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/range-perf.pl @@ -0,0 +1,137 @@ +#!/usr/bin/perl -w +# +# range-perf Extract a time range from Linux "perf script" output. +# +# USAGE EXAMPLE: +# +# perf record -F 100 -a -- sleep 60 +# perf script | ./perf2range.pl 10 20 # range 10 to 20 seconds only +# perf script | ./perf2range.pl 0 0.5 # first half second only +# +# MAKING A SERIES OF FLAME GRAPHS: +# +# Let's say you had the output of "perf script" in a file, out.stacks01, which +# was for a 180 second profile. The following command creates a series of +# flame graphs for each 10 second interval: +# +# for i in `seq 0 10 170`; do cat out.stacks01 | \ +# ./perf2range.pl $i $((i + 10)) | ./stackcollapse-perf.pl | \ +# grep -v cpu_idle | ./flamegraph.pl --hash --color=java \ +# --title="range $i $((i + 10))" > out.range_$i.svg; echo $i done; done +# +# In that example, I used "--color=java" for the Java palette, and excluded +# the idle CPU task. Customize as needed. +# +# Copyright 2017 Netflix, Inc. +# Licensed under the Apache License, Version 2.0 (the "License") +# +# 21-Feb-2017 Brendan Gregg Created this. + +use strict; +use Getopt::Long; +use POSIX 'floor'; + +sub usage { + die < \$timeraw, + 'timezerosecs' => \$timezerosecs, +) or usage(); + +if (@ARGV < 2 || $ARGV[0] eq "-h" || $ARGV[0] eq "--help") { + usage(); + exit; +} +my $begin = $ARGV[0]; +my $end = $ARGV[1]; + +# +# Parsing +# +# IP only examples: +# +# java 52025 [026] 99161.926202: cycles: +# java 14341 [016] 252732.474759: cycles: 7f36571947c0 nmethod::is_nmethod() const (/... +# java 14514 [022] 28191.353083: cpu-clock: 7f92b4fdb7d4 Ljava_util_List$size$0;::call (/tmp/perf-11936.map) +# swapper 0 [002] 6035557.056977: 10101010 cpu-clock: ffffffff810013aa xen_hypercall_sched_op+0xa (/lib/modules/4.9-virtual/build/vmlinux) +# bash 25370 603are 6036.991603: 10101010 cpu-clock: 4b931e [unknown] (/bin/bash) +# bash 25370/25370 6036036.799684: cpu-clock: 4b913b [unknown] (/bin/bash) +# other combinations are possible. +# +# Stack examples (-g): +# +# swapper 0 [021] 28648.467059: cpu-clock: +# ffffffff810013aa xen_hypercall_sched_op ([kernel.kallsyms]) +# ffffffff8101cb2f default_idle ([kernel.kallsyms]) +# ffffffff8101d406 arch_cpu_idle ([kernel.kallsyms]) +# ffffffff810bf475 cpu_startup_entry ([kernel.kallsyms]) +# ffffffff81010228 cpu_bringup_and_idle ([kernel.kallsyms]) +# +# java 14375 [022] 28648.467079: cpu-clock: +# 7f92bdd98965 Ljava/io/OutputStream;::write (/tmp/perf-11936.map) +# 7f8808cae7a8 [unknown] ([unknown]) +# +# swapper 0 [005] 5076.836336: cpu-clock: +# ffffffff81051586 native_safe_halt ([kernel.kallsyms]) +# ffffffff8101db4f default_idle ([kernel.kallsyms]) +# ffffffff8101e466 arch_cpu_idle ([kernel.kallsyms]) +# ffffffff810c2b31 cpu_startup_entry ([kernel.kallsyms]) +# ffffffff810427cd start_secondary ([kernel.kallsyms]) +# +# swapper 0 [002] 6034779.719110: 10101010 cpu-clock: +# 2013aa xen_hypercall_sched_op+0xfe20000a (/lib/modules/4.9-virtual/build/vmlinux) +# a72f0e default_idle+0xfe20001e (/lib/modules/4.9-virtual/build/vmlinux) +# 2392bf arch_cpu_idle+0xfe20000f (/lib/modules/4.9-virtual/build/vmlinux) +# a73333 default_idle_call+0xfe200023 (/lib/modules/4.9-virtual/build/vmlinux) +# 2c91a4 cpu_startup_entry+0xfe2001c4 (/lib/modules/4.9-virtual/build/vmlinux) +# 22b64a cpu_bringup_and_idle+0xfe20002a (/lib/modules/4.9-virtual/build/vmlinux) +# +# bash 25370/25370 6035935.188539: cpu-clock: +# b9218 [unknown] (/bin/bash) +# 2037fe8 [unknown] ([unknown]) +# other combinations are possible. +# +# This regexp matches the event line, and puts time in $1, and the event name +# in $2: +# +my $event_regexp = qr/ +([0-9\.]+): *\S* *(\S+):/; + +my $line; +my $start = 0; +my $ok = 0; +my $time; + +while (1) { + $line = ; + last unless defined $line; + next if $line =~ /^#/; # skip comments + + if ($line =~ $event_regexp) { + my ($ts, $event) = ($1, $2, $3); + $start = $ts if $start == 0; + + if ($timezerosecs) { + $time = $ts - floor($start); + } elsif (!$timeraw) { + $time = $ts - $start; + } else { + $time = $ts; # raw times + } + + $ok = 1 if $time >= $begin; + # assume samples are in time order: + exit if $time > $end; + } + + print $line if $ok; +} diff --git a/benchmarks/cli/_script/flamegraph/record-test.sh b/benchmarks/cli/_script/flamegraph/record-test.sh new file mode 100755 index 00000000000..569ecc966bb --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/record-test.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# record-test.sh - Overwrite flame graph test result files. +# +# See test.sh, which checks these resulting files. +# +# Currently only tests stackcollapse-perf.pl. + +set -v -x + +# ToDo: add some form of --inline, and --inline --context tests. These are +# tricky since they use addr2line, whose output will vary based on the test +# system's binaries and symbol tables. +for opt in pid tid kernel jit all addrs; do + for testfile in test/*.txt ; do + echo testing $testfile : $opt + outfile=${testfile#*/} + outfile=test/results/${outfile%.txt}"-collapsed-${opt}.txt" + ./stackcollapse-perf.pl --"${opt}" "${testfile}" 2> /dev/null > $outfile + done +done diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl new file mode 100755 index 00000000000..8456d56b918 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl @@ -0,0 +1,61 @@ +#!/usr/bin/perl -ws +# +# stackcollapse-aix Collapse AIX /usr/bin/procstack backtraces +# +# Parse a list of backtraces as generated with the poor man's aix-perf.pl +# profiler +# + +use strict; + +my $process = ""; +my $current = ""; +my $previous_function = ""; + +my %stacks; + +while(<>) { + chomp; + if (m/^\d+:/) { + if(!($current eq "")) { + $current = $process . ";" . $current; + $stacks{$current} += 1; + $current = ""; + } + m/^\d+: ([^ ]*)/; + $process = $1; + $current = ""; + } + elsif(m/^---------- tid# \d+/){ + if(!($current eq "")) { + $current = $process . ";" . $current; + $stacks{$current} += 1; + } + $current = ""; + } + elsif(m/^(0x[0-9abcdef]*) *([^ ]*) ([^ ]*) ([^ ]*)/) { + my $function = $2; + my $alt = $1; + $function=~s/\(.*\)?//; + if($function =~ /^\[.*\]$/) { + $function = $alt; + } + if ($current) { + $current = $function . ";" . $current; + } + else { + $current = $function; + } + } +} + +if(!($current eq "")) { + $current = $process . ";" . $current; + $stacks{$current} += 1; + $current = ""; + $process = ""; +} + +foreach my $k (sort { $a cmp $b } keys %stacks) { + print "$k $stacks{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl new file mode 100755 index 00000000000..f458c3e3af1 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl @@ -0,0 +1,72 @@ +#!/usr/bin/perl -w +# +# stackcollapse-bpftrace.pl collapse bpftrace samples into single lines. +# +# USAGE ./stackcollapse-bpftrace.pl infile > outfile +# +# Example input: +# +# @[ +# _raw_spin_lock_bh+0 +# tcp_recvmsg+808 +# inet_recvmsg+81 +# sock_recvmsg+67 +# sock_read_iter+144 +# new_sync_read+228 +# __vfs_read+41 +# vfs_read+142 +# sys_read+85 +# do_syscall_64+115 +# entry_SYSCALL_64_after_hwframe+61 +# ]: 3 +# +# Example output: +# +# entry_SYSCALL_64_after_hwframe+61;do_syscall_64+115;sys_read+85;vfs_read+142;__vfs_read+41;new_sync_read+228;sock_read_iter+144;sock_recvmsg+67;inet_recvmsg+81;tcp_recvmsg+808;_raw_spin_lock_bh+0 3 +# +# Copyright 2018 Peter Sanford. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# (http://www.gnu.org/copyleft/gpl.html) +# + +use strict; + +my @stack; +my $in_stack = 0; + +foreach (<>) { + chomp; + if (!$in_stack) { + if (/^@\[$/) { + $in_stack = 1; + } elsif (/^@\[,\s(.*)\]: (\d+)/) { + print $1 . " $2\n"; + } + } else { + if (m/^,?\s?(.*)\]: (\d+)/) { + if (length $1) { + push(@stack, $1); + } + print join(';', reverse(@stack)) . " $2\n"; + $in_stack = 0; + @stack = (); + } else { + $_ =~ s/^\s+//; + push(@stack, $_); + } + } +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py b/benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py new file mode 100755 index 00000000000..b4869781d90 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py @@ -0,0 +1,144 @@ +#!/usr/bin/python +# +# stackcolllapse-chrome-tracing.py collapse Trace Event Format [1] +# callstack events into single lines. +# +# [1] https://github.com/catapult-project/catapult/wiki/Trace-Event-Format +# +# USAGE: ./stackcollapse-chrome-tracing.py input_json [input_json...] > outfile +# +# Example input: +# +# {"traceEvents":[ +# {"pid":1,"tid":2,"ts":0,"ph":"X","name":"Foo","dur":50}, +# {"pid":1,"tid":2,"ts":10,"ph":"X","name":"Bar","dur":30} +# ]} +# +# Example output: +# +# Foo 20.0 +# Foo;Bar 30.0 +# +# Input may contain many stack trace events from many processes/threads. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 4-Jan-2018 Marcin Kolny Created this. +import argparse +import json + +stack_identifiers = {} + + +class Event: + def __init__(self, label, timestamp, dur): + self.label = label + self.timestamp = timestamp + self.duration = dur + self.total_duration = dur + + def get_stop_timestamp(self): + return self.timestamp + self.duration + + +def cantor_pairing(a, b): + s = a + b + return s * (s + 1) / 2 + b + + +def get_trace_events(trace_file, events_dict): + json_data = json.load(trace_file) + + for entry in json_data['traceEvents']: + if entry['ph'] == 'X': + cantor_val = cantor_pairing(int(entry['tid']), int(entry['pid'])) + if 'dur' not in entry: + continue + if cantor_val not in events_dict: + events_dict[cantor_val] = [] + events_dict[cantor_val].append(Event(entry['name'], float(entry['ts']), float(entry['dur']))) + + +def load_events(trace_files): + events = {} + + for trace_file in trace_files: + get_trace_events(trace_file, events) + + for key in events: + events[key].sort(key=lambda x: x.timestamp) + + return events + + +def save_stack(stack): + first = True + event = None + identifier = '' + + for event in stack: + if first: + first = False + else: + identifier += ';' + identifier += event.label + + if not event: + return + + if identifier in stack_identifiers: + stack_identifiers[identifier] += event.total_duration + else: + stack_identifiers[identifier] = event.total_duration + + +def load_stack_identifiers(events): + event_stack = [] + + for e in events: + if not event_stack: + event_stack.append(e) + else: + while event_stack and event_stack[-1].get_stop_timestamp() <= e.timestamp: + save_stack(event_stack) + event_stack.pop() + + if event_stack: + event_stack[-1].total_duration -= e.duration + + event_stack.append(e) + + while event_stack: + save_stack(event_stack) + event_stack.pop() + + +parser = argparse.ArgumentParser() +parser.add_argument('input_file', nargs='+', + type=argparse.FileType('r'), + help='Chrome Tracing input files') +args = parser.parse_args() + +all_events = load_events(args.input_file) +for tid_pid in all_events: + load_stack_identifiers(all_events[tid_pid]) + +for identifiers, duration in stack_identifiers.items(): + print(identifiers + ' ' + str(duration)) diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl new file mode 100755 index 00000000000..c5e6b17e44b --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl @@ -0,0 +1,98 @@ +#!/usr/bin/perl -w +# +# stackcollapse-elfutils Collapse elfutils stack (eu-stack) backtraces +# +# Parse a list of elfutils backtraces as generated with the poor man's +# profiler [1]: +# +# for x in $(seq 1 "$nsamples"); do +# eu-stack -p "$pid" "$@" +# sleep "$sleeptime" +# done +# +# [1] http://poormansprofiler.org/ +# +# Copyright 2014 Gabriel Corona. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +use strict; +use Getopt::Long; + +my $with_pid = 0; +my $with_tid = 0; + +GetOptions('pid' => \$with_pid, + 'tid' => \$with_tid) +or die < outfile\n + --pid # include PID + --tid # include TID +USAGE_END + +my $pid = ""; +my $tid = ""; +my $current = ""; +my $previous_function = ""; + +my %stacks; + +sub add_current { + if(!($current eq "")) { + my $entry; + if ($with_tid) { + $current = "TID=$tid;$current"; + } + if ($with_pid) { + $current = "PID=$pid;$current"; + } + $stacks{$current} += 1; + $current = ""; + } +} + +while(<>) { + chomp; + if (m/^PID ([0-9]*)/) { + add_current(); + $pid = $1; + } + elsif(m/^TID ([0-9]*)/) { + add_current(); + $tid = $1; + } elsif(m/^#[0-9]* *0x[0-9a-f]* (.*)/) { + if ($current eq "") { + $current = $1; + } else { + $current = "$1;$current"; + } + } elsif(m/^#[0-9]* *0x[0-9a-f]*/) { + if ($current eq "") { + $current = "[unknown]"; + } else { + $current = "[unknown];$current"; + } + } +} +add_current(); + +foreach my $k (sort { $a cmp $b } keys %stacks) { + print "$k $stacks{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl new file mode 100755 index 00000000000..4fe74ffa7b8 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl @@ -0,0 +1,61 @@ +#!/usr/bin/perl -ws +# +# stackcollapse-faulthandler Collapse Python faulthandler backtraces +# +# Parse a list of Python faulthandler backtraces as generated with +# faulthandler.dump_traceback_later. +# +# Copyright 2014 Gabriel Corona. All rights reserved. +# Copyright 2017 Jonathan Kolb. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +use strict; + +my $current = ""; + +my %stacks; + +while(<>) { + chomp; + if (m/^Thread/) { + $current="" + } + elsif(m/^ File "([^"]*)", line ([0-9]*) in (.*)/) { + my $function = $1 . ":" . $2 . ":" . $3; + if ($current eq "") { + $current = $function; + } else { + $current = $function . ";" . $current; + } + } elsif(!($current eq "")) { + $stacks{$current} += 1; + $current = ""; + } +} + +if(!($current eq "")) { + $stacks{$current} += 1; + $current = ""; +} + +foreach my $k (sort { $a cmp $b } keys %stacks) { + print "$k $stacks{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl new file mode 100755 index 00000000000..8e9831b22e5 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl @@ -0,0 +1,72 @@ +#!/usr/bin/perl -ws +# +# stackcollapse-gdb Collapse GDB backtraces +# +# Parse a list of GDB backtraces as generated with the poor man's +# profiler [1]: +# +# for x in $(seq 1 500); do +# gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $pid 2> /dev/null +# sleep 0.01 +# done +# +# [1] http://poormansprofiler.org/ +# +# Copyright 2014 Gabriel Corona. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +use strict; + +my $current = ""; +my $previous_function = ""; + +my %stacks; + +while(<>) { + chomp; + if (m/^Thread/) { + $current="" + } + elsif(m/^#[0-9]* *([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*)/) { + my $function = $3; + my $alt = $1; + if(not($1 =~ /0x[a-zA-Z0-9]*/)) { + $function = $alt; + } + if ($current eq "") { + $current = $function; + } else { + $current = $function . ";" . $current; + } + } elsif(!($current eq "")) { + $stacks{$current} += 1; + $current = ""; + } +} + +if(!($current eq "")) { + $stacks{$current} += 1; + $current = ""; +} + +foreach my $k (sort { $a cmp $b } keys %stacks) { + print "$k $stacks{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-go.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-go.pl new file mode 100755 index 00000000000..3b2ce3c552f --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-go.pl @@ -0,0 +1,150 @@ +#!/usr/bin/perl -w +# +# stackcollapse-go.pl collapse golang samples into single lines. +# +# Parses golang smaples generated by "go tool pprof" and outputs stacks as +# single lines, with methods separated by semicolons, and then a space and an +# occurrence count. For use with flamegraph.pl. +# +# USAGE: ./stackcollapse-go.pl infile > outfile +# +# Example Input: +# ... +# Samples: +# samples/count cpu/nanoseconds +# 1 10000000: 1 2 +# 2 10000000: 3 2 +# 1 10000000: 4 2 +# ... +# Locations +# 1: 0x58b265 scanblock :0 s=0 +# 2: 0x599530 GC :0 s=0 +# 3: 0x58a999 flushptrbuf :0 s=0 +# 4: 0x58d6a8 runtime.MSpan_Sweep :0 s=0 +# ... +# Mappings +# ... +# +# Example Output: +# +# GC;flushptrbuf 2 +# GC;runtime.MSpan_Sweep 1 +# GC;scanblock 1 +# +# Input may contain many stacks as generated from go tool pprof: +# +# go tool pprof -seconds=60 -raw -output=a.pprof http://$ADDR/debug/pprof/profile +# +# For format of text profile, See golang/src/internal/pprof/profile/profile.go +# +# Copyright 2017 Sijie Yang (yangsijie@baidu.com). All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# (http://www.gnu.org/copyleft/gpl.html) +# +# 16-Jan-2017 Sijie Yang Created this. + +use strict; + +use Getopt::Long; + +# tunables +my $help = 0; + +sub usage { + die < outfile\n +USAGE_END +} + +GetOptions( + 'help' => \$help, +) or usage(); +$help && usage(); + +# internals +my $state = "ignore"; +my %stacks; +my %frames; +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $stacks{$stack} += $count; +} + +# +# Output stack string in required format. For example, for the following samples, +# format_statck() would return GC;runtime.MSpan_Sweep for stack "4 2" +# +# Locations +# 1: 0x58b265 scanblock :0 s=0 +# 2: 0x599530 GC :0 s=0 +# 3: 0x58a999 flushptrbuf :0 s=0 +# 4: 0x58d6a8 runtime.MSpan_Sweep :0 s=0 +# +sub format_statck { + my ($stack) = @_; + my @loc_list = split(/ /, $stack); + + for (my $i=0; $i<=$#loc_list; $i++) { + my $loc_name = $frames{$loc_list[$i]}; + $loc_list[$i] = $loc_name if ($loc_name); + } + return join(";", reverse(@loc_list)); +} + +foreach (<>) { + next if m/^#/; + chomp; + + if ($state eq "ignore") { + if (/Samples:/) { + $state = "sample"; + next; + } + + } elsif ($state eq "sample") { + if (/^\s*([0-9]+)\s*[0-9]+: ([0-9 ]+)/) { + my $samples = $1; + my $stack = $2; + remember_stack($stack, $samples); + } elsif (/Locations/) { + $state = "location"; + next; + } + + } elsif ($state eq "location") { + if (/^\s*([0-9]*): 0x[0-9a-f]+ (M=[0-9]+ )?([^ ]+) .*/) { + my $loc_id = $1; + my $loc_name = $3; + $frames{$loc_id} = $loc_name; + } elsif (/Mappings/) { + $state = "mapping"; + last; + } + } +} + +foreach my $k (keys %stacks) { + my $stack = format_statck($k); + my $count = $stacks{$k}; + $collapsed{$stack} += $count; +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl new file mode 100644 index 00000000000..f8ffa8bd4d7 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl @@ -0,0 +1,145 @@ +#!/usr/bin/perl -w +# +# stackcollapse-ibmjava.pl collapse jstack samples into single lines. +# +# Parses Java stacks generated by IBM Java with methods separated by semicolons, +# and then a space and an occurrence count. +# +# USAGE: ./stackcollapse-ibmjava.pl infile > outfile +# +# Example input: +# +# NULL +# 1XMTHDINFO Thread Details +# NULL +# NULL +# 3XMTHREADINFO "Default Executor-thread-149164" J9VMThread:0x0000000008132B00, j9thread_t:0x000000001A810B90, java/lang/Thread:0x0000000712BE8E48, state:R, prio=5 +# 3XMJAVALTHREAD (java/lang/Thread getId:0x3493E, isDaemon:true) +# 3XMTHREADINFO1 (native thread ID:0x3158, native priority:0x5, native policy:UNKNOWN, vmstate:R, vm thread flags:0x00000001) +# 3XMCPUTIME CPU usage total: 0.421875000 secs, user: 0.343750000 secs, system: 0.078125000 secs, current category="Application" +# 3XMHEAPALLOC Heap bytes allocated since last GC cycle=0 (0x0) +# 3XMTHREADINFO3 Java callstack: +# 4XESTACKTRACE at java/net/SocketInputStream.socketRead0(Native Method) +# 4XESTACKTRACE at java/net/SocketInputStream.socketRead(SocketInputStream.java:127(Compiled Code)) +# 4XESTACKTRACE at java/net/SocketInputStream.read(SocketInputStream.java:182(Compiled Code)) +# 4XESTACKTRACE at java/net/SocketInputStream.read(SocketInputStream.java:152(Compiled Code)) +# 4XESTACKTRACE at java/io/FilterInputStream.read(FilterInputStream.java:144(Compiled Code)) +# ... +# 4XESTACKTRACE at java/lang/Thread.run(Thread.java:785(Compiled Code)) +# +# Example output: +# +# Default Executor-thread-149164;java/lang/Thread.run;java/net/SocketInputStream/read;java/net/SocketInputStream.socketRead0 1 +# +# +# Copyright 2014 Federico Juinio. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# (http://www.gnu.org/copyleft/gpl.html) +# +# 23-Aug-2023 Federico Juinio created this based from stackcollapse-jstack.pl + +use strict; + +use Getopt::Long; + +# tunables +my $include_tname = 1; # include thread names in stacks +my $include_tid = 0; # include thread IDs in stacks +my $shorten_pkgs = 0; # shorten package names +my $help = 0; + +sub usage { + die < outfile\n + --include-tname + --no-include-tname # include/omit thread names in stacks (default: include) + --include-tid + --no-include-tid # include/omit thread IDs in stacks (default: omit) + --shorten-pkgs + --no-shorten-pkgs # (don't) shorten package names (default: don't shorten) + + eg, + $0 --no-include-tname stacks.txt > collapsed.txt +USAGE_END +} + +GetOptions( + 'include-tname!' => \$include_tname, + 'include-tid!' => \$include_tid, + 'shorten-pkgs!' => \$shorten_pkgs, + 'help' => \$help, +) or usage(); +$help && usage(); + + +# internals +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} + +my @stack; +my $tname; +my $state = "?"; + +foreach (<>) { + next if m/^#/; + chomp; + + if (m/^3XMTHREADINFO3 Native callstack:/) { + # save stack + if (defined $tname) { unshift @stack, $tname; } + remember_stack(join(";", @stack), 1) if @stack; + undef @stack; + undef $tname; + $state = "?"; + next; + } + + # look for thread header line and parse thread name and state + if (/^3XMTHREADINFO "([^"]*).* state:(.*), /) { + my $name = $1; + if ($include_tname) { + $tname = $name; + } + $state = $2; + # special handling for "Anonymous native threads" + } elsif (/3XMTHREADINFO Anonymous native thread/) { + $tname = "Anonymous native thread"; + # look for thread id + } elsif (/^3XMTHREADINFO1 \(native thread ID:([^ ]*), native priority/) { + if ($include_tname && $include_tid) { + $tname = $tname . "-" . $1 + } + # collect stack frames + } elsif (/^4XESTACKTRACE at ([^\(]*)/) { + my $func = $1; + if ($shorten_pkgs) { + my ($pkgs, $clsFunc) = ( $func =~ m/(.*\.)([^.]+\.[^.]+)$/ ); + $pkgs =~ s/(\w)\w*/$1/g; + $func = $pkgs . $clsFunc; + } + unshift @stack, $func; + + } +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl new file mode 100755 index 00000000000..3cbaa87bc27 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl @@ -0,0 +1,34 @@ +#!/usr/bin/perl -w +# +# stackcollapse-instruments.pl +# +# Parses a file containing a call tree as produced by XCode Instruments +# (Edit > Deep Copy) and produces output suitable for flamegraph.pl. +# +# USAGE: ./stackcollapse-instruments.pl infile > outfile + +use strict; + +my @stack = (); + +<>; +foreach (<>) { + chomp; + /\d+\.\d+ (?:min|s|ms)\s+\d+\.\d+%\s+(\d+(?:\.\d+)?) (min|s|ms)\t \t(\s*)(.+)/ or die; + my $func = $4; + my $depth = length ($3); + $stack [$depth] = $4; + foreach my $i (0 .. $depth - 1) { + print $stack [$i]; + print ";"; + } + + my $time = 0 + $1; + if ($2 eq "min") { + $time *= 60*1000; + } elsif ($2 eq "s") { + $time *= 1000; + } + + printf("%s %.0f\n", $func, $time); +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl new file mode 100755 index 00000000000..19badbca6cb --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl @@ -0,0 +1,72 @@ +#!/usr/bin/perl -w +# +# stackcolllapse-java-exceptions.pl collapse java exceptions (found in logs) into single lines. +# +# Parses Java error stacks found in a log file and outputs them as +# single lines, with methods separated by semicolons, and then a space and an +# occurrence count. Inspired by stackcollapse-jstack.pl except that it does +# not act as a performance profiler. +# +# It can be useful if a Java process dumps a lot of different stacks in its logs +# and you want to quickly identify the biggest culprits. +# +# USAGE: ./stackcollapse-java-exceptions.pl infile > outfile +# +# Copyright 2018 Paul de Verdiere. All rights reserved. + +use strict; +use Getopt::Long; + +# tunables +my $shorten_pkgs = 0; # shorten package names +my $no_pkgs = 0; # really shorten package names!! +my $help = 0; + +sub usage { + die < outfile\n + --shorten-pkgs : shorten package names + --no-pkgs : suppress package names (makes SVG much more readable) + +USAGE_END +} + +GetOptions( + 'shorten-pkgs!' => \$shorten_pkgs, + 'no-pkgs!' => \$no_pkgs, + 'help' => \$help, +) or usage(); +$help && usage(); + +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} + +my @stack; + +foreach (<>) { + chomp; + + if (/^\s*at ([^\(]*)/) { + my $func = $1; + if ($shorten_pkgs || $no_pkgs) { + my ($pkgs, $clsFunc) = ( $func =~ m/(.*\.)([^.]+\.[^.]+)$/ ); + $pkgs =~ s/(\w)\w*/$1/g; + $func = $no_pkgs ? $clsFunc: $pkgs . $clsFunc; + } + unshift @stack, $func; + } elsif (@stack ) { + next if m/.*waiting on .*/; + remember_stack(join(";", @stack), 1) if @stack; + undef @stack; + } +} + +remember_stack(join(";", @stack), 1) if @stack; + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl new file mode 100755 index 00000000000..da5740b6ee6 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl @@ -0,0 +1,176 @@ +#!/usr/bin/perl -w +# +# stackcollapse-jstack.pl collapse jstack samples into single lines. +# +# Parses Java stacks generated by jstack(1) and outputs RUNNABLE stacks as +# single lines, with methods separated by semicolons, and then a space and an +# occurrence count. This also filters some other "RUNNABLE" states that we +# know are probably not running, such as epollWait. For use with flamegraph.pl. +# +# You want this to process the output of at least 100 jstack(1)s. ie, run it +# 100 times with a sleep interval, and append to a file. This is really a poor +# man's Java profiler, due to the overheads of jstack(1), and how it isn't +# capturing stacks asynchronously. For a better profiler, see: +# http://www.brendangregg.com/blog/2014-06-12/java-flame-graphs.html +# +# USAGE: ./stackcollapse-jstack.pl infile > outfile +# +# Example input: +# +# "MyProg" #273 daemon prio=9 os_prio=0 tid=0x00007f273c038800 nid=0xe3c runnable [0x00007f28a30f2000] +# java.lang.Thread.State: RUNNABLE +# at java.net.SocketInputStream.socketRead0(Native Method) +# at java.net.SocketInputStream.read(SocketInputStream.java:121) +# ... +# at java.lang.Thread.run(Thread.java:744) +# +# Example output: +# +# MyProg;java.lang.Thread.run;java.net.SocketInputStream.read;java.net.SocketInputStream.socketRead0 1 +# +# Input may be created and processed using: +# +# i=0; while (( i++ < 200 )); do jstack PID >> out.jstacks; sleep 10; done +# cat out.jstacks | ./stackcollapse-jstack.pl > out.stacks-folded +# +# WARNING: jstack(1) incurs overheads. Test before use, or use a real profiler. +# +# Copyright 2014 Brendan Gregg. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# (http://www.gnu.org/copyleft/gpl.html) +# +# 14-Sep-2014 Brendan Gregg Created this. + +use strict; + +use Getopt::Long; + +# tunables +my $include_tname = 1; # include thread names in stacks +my $include_tid = 0; # include thread IDs in stacks +my $shorten_pkgs = 0; # shorten package names +my $help = 0; + +sub usage { + die < outfile\n + --include-tname + --no-include-tname # include/omit thread names in stacks (default: include) + --include-tid + --no-include-tid # include/omit thread IDs in stacks (default: omit) + --shorten-pkgs + --no-shorten-pkgs # (don't) shorten package names (default: don't shorten) + + eg, + $0 --no-include-tname stacks.txt > collapsed.txt +USAGE_END +} + +GetOptions( + 'include-tname!' => \$include_tname, + 'include-tid!' => \$include_tid, + 'shorten-pkgs!' => \$shorten_pkgs, + 'help' => \$help, +) or usage(); +$help && usage(); + + +# internals +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} + +my @stack; +my $tname; +my $state = "?"; + +foreach (<>) { + next if m/^#/; + chomp; + + if (m/^$/) { + # only include RUNNABLE states + goto clear if $state ne "RUNNABLE"; + + # save stack + if (defined $tname) { unshift @stack, $tname; } + remember_stack(join(";", @stack), 1) if @stack; +clear: + undef @stack; + undef $tname; + $state = "?"; + next; + } + + # + # While parsing jstack output, the $state variable may be altered from + # RUNNABLE to other states. This causes the stacks to be filtered later, + # since only RUNNABLE stacks are included. + # + + if (/^"([^"]*)/) { + my $name = $1; + + if ($include_tname) { + $tname = $name; + unless ($include_tid) { + $tname =~ s/-\d+$//; + } + } + + # set state for various background threads + $state = "BACKGROUND" if $name =~ /C. CompilerThread/; + $state = "BACKGROUND" if $name =~ /Signal Dispatcher/; + $state = "BACKGROUND" if $name =~ /Service Thread/; + $state = "BACKGROUND" if $name =~ /Attach Listener/; + + } elsif (/java.lang.Thread.State: (\S+)/) { + $state = $1 if $state eq "?"; + } elsif (/^\s*at ([^\(]*)/) { + my $func = $1; + if ($shorten_pkgs) { + my ($pkgs, $clsFunc) = ( $func =~ m/(.*\.)([^.]+\.[^.]+)$/ ); + $pkgs =~ s/(\w)\w*/$1/g; + $func = $pkgs . $clsFunc; + } + unshift @stack, $func; + + # fix state for epollWait + $state = "WAITING" if $func =~ /epollWait/; + $state = "WAITING" if $func =~ /EPoll\.wait/; + + + # fix state for various networking functions + $state = "NETWORK" if $func =~ /socketAccept$/; + $state = "NETWORK" if $func =~ /Socket.*accept0$/; + $state = "NETWORK" if $func =~ /socketRead0$/; + + } elsif (/^\s*-/ or /^2\d\d\d-/ or /^Full thread dump/ or + /^JNI global references:/) { + # skip these info lines + next; + } else { + warn "Unrecognized line: $_"; + } +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk new file mode 100755 index 00000000000..59aaae3d73b --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk @@ -0,0 +1,74 @@ +#!/usr/bin/awk -f +# +# stackcollapse-ljp.awk collapse lightweight java profile reports +# into single lines stacks. +# +# Parses a list of multiline stacks generated by: +# +# https://code.google.com/p/lightweight-java-profiler +# +# and outputs a semicolon separated stack followed by a space and a count. +# +# USAGE: ./stackcollapse-ljp.pl infile > outfile +# +# Example input: +# +# 42 3 my_func_b(prog.java:455) +# my_func_a(prog.java:123) +# java.lang.Thread.run(Thread.java:744) +# [...] +# +# Example output: +# +# java.lang.Thread.run;my_func_a;my_func_b 42 +# +# The unused number is the number of frames in each stack. +# +# Copyright 2014 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 12-Jun-2014 Brendan Gregg Created this. + +$1 == "Total" { + # We're done. Print last stack and exit. + print stack, count + exit +} + +{ + # Strip file location. Comment this out to keep. + gsub(/\(.*\)/, "") +} + +NF == 3 { + # New stack begins. Print previous buffered stack. + if (count) + print stack, count + + # Begin a new stack. + count = $1 + stack = $3 +} + +NF == 1 { + # Build stack. + stack = $1 ";" stack +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk new file mode 100755 index 00000000000..e1a122d459e --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk @@ -0,0 +1,228 @@ +#!/usr/bin/awk -f + +# +# This program generates collapsed off-cpu stacks fit for use by flamegraph.pl +# from scheduler data collected via perf_events. +# +# Outputs the cumulative time off cpu in us for each distinct stack observed. +# +# Some awk variables further control behavior: +# +# record_tid If truthy, causes all stack traces to include the +# command and LWP id. +# +# record_wake_stack If truthy, stacks include the frames from the wakeup +# event in addition to the sleep event. +# See http://www.brendangregg.com/FlameGraphs/offcpuflamegraphs.html#Wakeup +# +# recurse If truthy, attempt to recursively identify and +# visualize the full wakeup stack chain. +# See http://www.brendangregg.com/FlameGraphs/offcpuflamegraphs.html#ChainGraph +# +# Note that this is only an approximation, as only the +# last sleep event is recorded (e.g. if a thread slept +# multiple times before waking another thread, only the +# last sleep event is used). Implies record_wake_stack=1 +# +# To set any of these variables from the command line, run via: +# +# stackcollapse-perf-sched.awk -v recurse=1 +# +# == Important warning == +# +# WARNING: tracing all scheduler events is very high overhead in perf. Even +# more alarmingly, there appear to be bugs in perf that prevent it from reliably +# getting consistent traces (even with large trace buffers), causing it to +# produce empty perf.data files with error messages of the form: +# +# 0x952790 [0x736d]: failed to process type: 3410 +# +# This failure is not determinisitic, so re-executing perf record will +# eventually succeed. +# +# == Usage == +# +# First, record data via perf_events: +# +# sudo perf record -g -e 'sched:sched_switch' \ +# -e 'sched:sched_stat_sleep' -e 'sched:sched_stat_blocked' \ +# -p -o perf.data -- sleep 1 +# +# Then post process with this script: +# +# sudo perf script -f time,comm,pid,tid,event,ip,sym,dso,trace -i perf.data | \ +# stackcollapse-perf-sched.awk -v recurse=1 | \ +# flamegraph.pl --color=io --countname=us >out.svg +# + +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2015 by MemSQL. All rights reserved. +# + +# +# Match a perf captured variable, returning just the contents. For example, for +# the following line, get_perf_captured_variable("pid") would return "27235": +# +# swapper 0 [006] 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns +# +function get_perf_captured_variable(variable) +{ + match($0, variable "=[^[:space:]]+") + return substr($0, RSTART + length(variable) + 1, + RLENGTH - length(variable) - 1) +} + +# +# The timestamp is the first field that ends in a colon, e.g.: +# +# swapper 0 [006] 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns +# +# or +# +# swapper 0/0 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns] +# +function get_perf_timestamp() +{ + match($0, " [^ :]+:") + return substr($0, RSTART + 1, RLENGTH - 2) +} + +!/^#/ && /sched:sched_switch/ { + switchcommand = get_perf_captured_variable("comm") + + switchpid = get_perf_captured_variable("prev_pid") + + switchtime=get_perf_timestamp() + + switchstack="" +} + +# +# Strip the function name from a stack entry +# +# Stack entry is expected to be formatted like: +# c60849 MyClass::Foo(unsigned long) (/home/areece/a.out) +# +function get_function_name() +{ + # We start from 2 since we don't need the hex offset. + # We stop at NF - 1 since we don't need the library path. + funcname = $2 + for (i = 3; i <= NF - 1; i++) { + funcname = funcname " " $i + } + return funcname +} + +(switchpid != 0 && /^\s/) { + if (switchstack == "") { + switchstack = get_function_name() + } else { + switchstack = get_function_name() ";" switchstack + } +} + +(switchpid != 0 && /^$/) { + switch_stacks[switchpid] = switchstack + delete last_switch_stacks[switchpid] + switch_time[switchpid] = switchtime + + switchpid=0 + switchcommand="" + switchstack="" +} + +!/^#/ && (/sched:sched_stat_sleep/ || /sched:sched_stat_blocked/) { + wakecommand=$1 + wakepid=$2 + + waketime=get_perf_timestamp() + + stat_next_command = get_perf_captured_variable("comm") + + stat_next_pid = get_perf_captured_variable("pid") + + stat_delay_ns = int(get_perf_captured_variable("delay")) + + wakestack="" +} + +(stat_next_pid != 0 && /^\s/) { + if (wakestack == "") { + wakestack = get_function_name() + } else { + # We build the wakestack in reverse order. + wakestack = wakestack ";" get_function_name() + } +} + +(stat_next_pid != 0 && /^$/) { + # + # For some reason, perf appears to output duplicate + # sched:sched_stat_sleep and sched:sched_stat_blocked events. We only + # handle the first event. + # + if (stat_next_pid in switch_stacks) { + last_wake_time[stat_next_pid] = waketime + + stack = switch_stacks[stat_next_pid] + if (recurse || record_wake_stack) { + stack = stack ";" wakestack + if (record_tid) { + stack = stack ";" wakecommand "-" wakepid + } else { + stack = stack ";" wakecommand + } + } + + if (recurse) { + if (last_wake_time[wakepid] > last_switch_time[stat_next_pid]) { + stack = stack ";-;" last_switch_stacks[wakepid] + } + last_switch_stacks[stat_next_pid] = stack + } + + delete switch_stacks[stat_next_pid] + + if (record_tid) { + stack_times[stat_next_command "-" stat_next_pid ";" stack] += stat_delay_ns + } else { + stack_times[stat_next_command ";" stack] += stat_delay_ns + } + } + + wakecommand="" + wakepid=0 + stat_next_pid=0 + stat_next_command="" + stat_delay_ms=0 +} + +END { + for (stack in stack_times) { + if (int(stack_times[stack] / 1000) > 0) { + print stack, int(stack_times[stack] / 1000) + } + } +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl new file mode 100755 index 00000000000..3ff39bfb87f --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl @@ -0,0 +1,435 @@ +#!/usr/bin/perl -w +# +# stackcollapse-perf.pl collapse perf samples into single lines. +# +# Parses a list of multiline stacks generated by "perf script", and +# outputs a semicolon separated stack followed by a space and a count. +# If memory addresses (+0xd) are present, they are stripped, and resulting +# identical stacks are colased with their counts summed. +# +# USAGE: ./stackcollapse-perf.pl [options] infile > outfile +# +# Run "./stackcollapse-perf.pl -h" to list options. +# +# Example input: +# +# swapper 0 [000] 158665.570607: cpu-clock: +# ffffffff8103ce3b native_safe_halt ([kernel.kallsyms]) +# ffffffff8101c6a3 default_idle ([kernel.kallsyms]) +# ffffffff81013236 cpu_idle ([kernel.kallsyms]) +# ffffffff815bf03e rest_init ([kernel.kallsyms]) +# ffffffff81aebbfe start_kernel ([kernel.kallsyms].init.text) +# [...] +# +# Example output: +# +# swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 1 +# +# Input may be created and processed using: +# +# perf record -a -g -F 997 sleep 60 +# perf script | ./stackcollapse-perf.pl > out.stacks-folded +# +# The output of "perf script" should include stack traces. If these are missing +# for you, try manually selecting the perf script output; eg: +# +# perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace | ... +# +# This is also required for the --pid or --tid options, so that the output has +# both the PID and TID. +# +# Copyright 2012 Joyent, Inc. All rights reserved. +# Copyright 2012 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 02-Mar-2012 Brendan Gregg Created this. +# 02-Jul-2014 " " Added process name to stacks. + +use strict; +use Getopt::Long; + +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} +my $annotate_kernel = 0; # put an annotation on kernel function +my $annotate_jit = 0; # put an annotation on jit symbols +my $annotate_all = 0; # enale all annotations +my $include_pname = 1; # include process names in stacks +my $include_pid = 0; # include process ID with process name +my $include_tid = 0; # include process & thread ID with process name +my $include_addrs = 0; # include raw address where a symbol can't be found +my $tidy_java = 1; # condense Java signatures +my $tidy_generic = 1; # clean up function names a little +my $target_pname; # target process name from perf invocation +my $event_filter = ""; # event type filter, defaults to first encountered event +my $event_defaulted = 0; # whether we defaulted to an event (none provided) +my $event_warning = 0; # if we printed a warning for the event + +my $show_inline = 0; +my $show_context = 0; + +my $srcline_in_input = 0; # if there are extra lines with source location (perf script -F+srcline) +GetOptions('inline' => \$show_inline, + 'context' => \$show_context, + 'srcline' => \$srcline_in_input, + 'pid' => \$include_pid, + 'kernel' => \$annotate_kernel, + 'jit' => \$annotate_jit, + 'all' => \$annotate_all, + 'tid' => \$include_tid, + 'addrs' => \$include_addrs, + 'event-filter=s' => \$event_filter) +or die < outfile\n + --pid # include PID with process names [1] + --tid # include TID and PID with process names [1] + --inline # un-inline using addr2line + --all # all annotations (--kernel --jit) + --kernel # annotate kernel functions with a _[k] + --jit # annotate jit functions with a _[j] + --context # adds source context to --inline + --srcline # parses output of 'perf script -F+srcline' and adds source context + --addrs # include raw addresses where symbols can't be found + --event-filter=EVENT # event name filter\n +[1] perf script must emit both PID and TIDs for these to work; eg, Linux < 4.1: + perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace + for Linux >= 4.1: + perf script -F comm,pid,tid,cpu,time,event,ip,sym,dso,trace + If you save this output add --header on Linux >= 3.14 to include perf info. +USAGE_END + +if ($annotate_all) { + $annotate_kernel = $annotate_jit = 1; +} + +my %inlineCache; + +my %nmCache; + +sub inlineCacheAdd { + my ($pc, $mod, $result) = @_; + if (defined($inlineCache{$pc})) { + $inlineCache{$pc}{$mod} = $result; + } else { + $inlineCache{$pc} = {$mod => $result}; + } +} + +# for the --inline option +sub inline { + my ($pc, $rawfunc, $mod) = @_; + + return $inlineCache{$pc}{$mod} if defined($inlineCache{$pc}{$mod}); + + # capture addr2line output + my $a2l_output = `addr2line -a $pc -e $mod -i -f -s -C`; + + # remove first line + $a2l_output =~ s/^(.*\n){1}//; + + if ($a2l_output =~ /\?\?\n\?\?:0/) { + # if addr2line fails and rawfunc is func+offset, then fall back to it + if ($rawfunc =~ /^(.+)\+0x([0-9a-f]+)$/) { + my $func = $1; + my $addr = hex $2; + + $nmCache{$mod}=`nm $mod` unless defined $nmCache{$mod}; + + if ($nmCache{$mod} =~ /^([0-9a-f]+) . \Q$func\E$/m) { + my $base = hex $1; + my $newPc = sprintf "0x%x", $base+$addr; + my $result = inline($newPc, '', $mod); + inlineCacheAdd($pc, $mod, $result); + return $result; + } + } + } + + my @fullfunc; + my $one_item = ""; + for (split /^/, $a2l_output) { + chomp $_; + + # remove discriminator info if exists + $_ =~ s/ \(discriminator \S+\)//; + + if ($one_item eq "") { + $one_item = $_; + } else { + if ($show_context == 1) { + unshift @fullfunc, $one_item . ":$_"; + } else { + unshift @fullfunc, $one_item; + } + $one_item = ""; + } + } + + my $result = join ";" , @fullfunc; + + inlineCacheAdd($pc, $mod, $result); + + return $result; +} + +my @stack; +my $pname; +my $m_pid; +my $m_tid; +my $m_period; + +# +# Main loop +# +while (defined($_ = <>)) { + + # find the name of the process launched by perf, by stepping backwards + # over the args to find the first non-option (no dash): + if (/^# cmdline/) { + my @args = split ' ', $_; + foreach my $arg (reverse @args) { + if ($arg !~ /^-/) { + $target_pname = $arg; + $target_pname =~ s:.*/::; # strip pathname + last; + } + } + } + + # skip remaining comments + next if m/^#/; + chomp; + + # end of stack. save cached data. + if (m/^$/) { + # ignore filtered samples + next if not $pname; + + if ($include_pname) { + if (defined $pname) { + unshift @stack, $pname; + } else { + unshift @stack, ""; + } + } + remember_stack(join(";", @stack), $m_period) if @stack; + undef @stack; + undef $pname; + next; + } + + # + # event record start + # + if (/^(\S.+?)\s+(\d+)\/*(\d+)*\s+/) { + # default "perf script" output has TID but not PID + # eg, "java 25607 4794564.109216: 1 cycles:" + # eg, "java 12688 [002] 6544038.708352: 235 cpu-clock:" + # eg, "V8 WorkerThread 25607 4794564.109216: 104345 cycles:" + # eg, "java 24636/25607 [000] 4794564.109216: 1 cycles:" + # eg, "java 12688/12764 6544038.708352: 10309278 cpu-clock:" + # eg, "V8 WorkerThread 24636/25607 [000] 94564.109216: 100 cycles:" + # other combinations possible + my ($comm, $pid, $tid, $period) = ($1, $2, $3, ""); + if (not $tid) { + $tid = $pid; + $pid = "?"; + } + + if (/:\s*(\d+)*\s+(\S+):\s*$/) { + $period = $1; + my $event = $2; + + if ($event_filter eq "") { + # By default only show events of the first encountered + # event type. Merging together different types, such as + # instructions and cycles, produces misleading results. + $event_filter = $event; + $event_defaulted = 1; + } elsif ($event ne $event_filter) { + if ($event_defaulted and $event_warning == 0) { + # only print this warning if necessary: + # when we defaulted and there was + # multiple event types. + print STDERR "Filtering for events of type: $event\n"; + $event_warning = 1; + } + next; + } + } + + if (not $period) { + $period = 1 + } + ($m_pid, $m_tid, $m_period) = ($pid, $tid, $period); + + if ($include_tid) { + $pname = "$comm-$m_pid/$m_tid"; + } elsif ($include_pid) { + $pname = "$comm-$m_pid"; + } else { + $pname = "$comm"; + } + $pname =~ tr/ /_/; + + # + # stack line + # + } elsif (/^\s*(\w+)\s*(.+) \((.*)\)/) { + # ignore filtered samples + next if not $pname; + + my ($pc, $rawfunc, $mod) = ($1, $2, $3); + + if ($show_inline == 1 && $mod !~ m/(perf-\d+.map|kernel\.|\[[^\]]+\])/) { + my $inlineRes = inline($pc, $rawfunc, $mod); + # - empty result this happens e.g., when $mod does not exist or is a path to a compressed kernel module + # if this happens, the user will see error message from addr2line written to stderr + # - if addr2line results in "??" , then it's much more sane to fall back than produce a '??' in graph + if($inlineRes ne "" and $inlineRes ne "??" and $inlineRes ne "??:??:0" ) { + unshift @stack, $inlineRes; + next; + } + } + + # Linux 4.8 included symbol offsets in perf script output by default, eg: + # 7fffb84c9afc cpu_startup_entry+0x800047c022ec ([kernel.kallsyms]) + # strip these off: + $rawfunc =~ s/\+0x[\da-f]+$//; + + next if $rawfunc =~ /^\(/; # skip process names + + my $is_unknown=0; + my @inline; + for (split /\->/, $rawfunc) { + my $func = $_; + + if ($func eq "[unknown]") { + if ($mod ne "[unknown]") { # use module name instead, if known + $func = $mod; + $func =~ s/.*\///; + } else { + $func = "unknown"; + $is_unknown=1; + } + + if ($include_addrs) { + $func = "\[$func \<$pc\>\]"; + } else { + $func = "\[$func\]"; + } + } + + if ($tidy_generic) { + $func =~ s/;/:/g; + if ($func !~ m/\.\(.*\)\./) { + # This doesn't look like a Go method name (such as + # "net/http.(*Client).Do"), so everything after the first open + # paren (that is not part of an "(anonymous namespace)") is + # just noise. + $func =~ s/\((?!anonymous namespace\)).*//; + } + # now tidy this horrible thing: + # 13a80b608e0a RegExp:[&<>\"\'] (/tmp/perf-7539.map) + $func =~ tr/"\'//d; + # fall through to $tidy_java + } + + if ($tidy_java and $pname =~ m/^java/) { + # along with $tidy_generic, converts the following: + # Lorg/mozilla/javascript/ContextFactory;.call(Lorg/mozilla/javascript/ContextAction;)Ljava/lang/Object; + # Lorg/mozilla/javascript/ContextFactory;.call(Lorg/mozilla/javascript/C + # Lorg/mozilla/javascript/MemberBox;.(Ljava/lang/reflect/Method;)V + # into: + # org/mozilla/javascript/ContextFactory:.call + # org/mozilla/javascript/ContextFactory:.call + # org/mozilla/javascript/MemberBox:.init + $func =~ s/^L// if $func =~ m:/:; + } + + # + # Annotations + # + # detect inlined from the @inline array + # detect kernel from the module name; eg, frames to parse include: + # ffffffff8103ce3b native_safe_halt ([kernel.kallsyms]) + # 8c3453 tcp_sendmsg (/lib/modules/4.3.0-rc1-virtual/build/vmlinux) + # 7d8 ipv4_conntrack_local+0x7f8f80b8 ([nf_conntrack_ipv4]) + # detect jit from the module name; eg: + # 7f722d142778 Ljava/io/PrintStream;::print (/tmp/perf-19982.map) + if (scalar(@inline) > 0) { + $func .= "_[i]" unless $func =~ m/\_\[i\]/; # inlined + } elsif ($annotate_kernel == 1 && $mod =~ m/(^\[|vmlinux$)/ && $mod !~ /unknown/) { + $func .= "_[k]"; # kernel + } elsif ($annotate_jit == 1 && $mod =~ m:/tmp/perf-\d+\.map:) { + $func .= "_[j]" unless $func =~ m/\_\[j\]/; # jitted + } + + # + # Source lines + # + # + # Sample outputs: + # | a.out 35081 252436.005167: 667783 cycles: + # | 408ebb some_method_name+0x8b (/full/path/to/a.out) + # | uniform_int_dist.h:300 + # | 4069f5 main+0x935 (/full/path/to/a.out) + # | file.cpp:137 + # | 7f6d2148eb25 __libc_start_main+0xd5 (/lib64/libc-2.33.so) + # | libc-2.33.so[27b25] + # + # | a.out 35081 252435.738165: 306459 cycles: + # | 7f6d213c2750 [unknown] (/usr/lib64/libkmod.so.2.3.6) + # | libkmod.so.2.3.6[6750] + # + # | a.out 35081 252435.738373: 315813 cycles: + # | 7f6d215ca51b __strlen_avx2+0x4b (/lib64/libc-2.33.so) + # | libc-2.33.so[16351b] + # | 7ffc71ee9580 [unknown] ([unknown]) + # | + # + # | a.out 35081 252435.718940: 247984 cycles: + # | ffffffff814f9302 up_write+0x32 ([kernel.kallsyms]) + # | [kernel.kallsyms][ffffffff814f9302] + if($srcline_in_input and not $is_unknown){ + $_ = <>; + chomp; + s/\[.*?\]//g; + s/^\s*//g; + s/\s*$//g; + $func.=':'.$_ unless $_ eq ""; + } + + push @inline, $func; + } + + unshift @stack, @inline; + } else { + warn "Unrecognized line: $_"; + } +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl new file mode 100755 index 00000000000..5bd7c2dada4 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl @@ -0,0 +1,74 @@ +#!/usr/bin/env perl +# +# Copyright (c) 2014 Ed Maste. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# stackcollapse-pmc.pl collapse hwpmc samples into single lines. +# +# Parses a list of multiline stacks generated by "hwpmc -G", and outputs a +# semicolon-separated stack followed by a space and a count. +# +# Usage: +# pmcstat -S unhalted-cycles -O pmc.out +# pmcstat -R pmc.out -z16 -G pmc.graph +# stackcollapse-pmc.pl pmc.graph > pmc.stack +# +# Example input: +# +# 03.07% [17] witness_unlock @ /boot/kernel/kernel +# 70.59% [12] __mtx_unlock_flags +# 16.67% [2] selfdfree +# 100.0% [2] sys_poll +# 100.0% [2] amd64_syscall +# 08.33% [1] pmap_ts_referenced +# 100.0% [1] vm_pageout +# 100.0% [1] fork_exit +# ... +# +# Example output: +# +# amd64_syscall;sys_poll;selfdfree;__mtx_unlock_flags;witness_unlock 2 +# amd64_syscall;sys_poll;pmap_ts_referenced;__mtx_unlock_flagsgeout;fork_exit 1 +# ... + +use warnings; +use strict; + +my @stack; +my $prev_count; +my $prev_indent = -1; + +while (defined($_ = <>)) { + if (m/^( *)[0-9.]+% \[([0-9]+)\]\s*(\S+)/) { + my $indent = length($1); + if ($indent <= $prev_indent) { + print join(';', reverse(@stack[0 .. $prev_indent])) . + " $prev_count\n"; + } + $stack[$indent] = $3; + $prev_count = $2; + $prev_indent = $indent; + } +} +print join(';', reverse(@stack[0 .. $prev_indent])) . " $prev_count\n"; diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl new file mode 100755 index 00000000000..9eae54592c4 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl @@ -0,0 +1,60 @@ +#!/usr/bin/perl -ws +# +# stackcollapse-recursive Collapse direct recursive backtraces +# +# Post-process a stack list and merge direct recursive calls: +# +# Example input: +# +# main;recursive;recursive;recursive;helper 1 +# +# Output: +# +# main;recursive;helper 1 +# +# Copyright 2014 Gabriel Corona. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +my %stacks; + +while(<>) { + chomp; + my ($stack_, $value) = (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + if ($stack_) { + my @stack = split(/;/, $stack_); + + my @result = (); + my $i; + my $last=""; + for($i=0; $i!=@stack; ++$i) { + if(!($stack[$i] eq $last)) { + $result[@result] = $stack[$i]; + $last = $stack[$i]; + } + } + + $stacks{join(";", @result)} += $value; + } +} + +foreach my $k (sort { $a cmp $b } keys %stacks) { + print "$k $stacks{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk new file mode 100755 index 00000000000..bafc4af3468 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk @@ -0,0 +1,231 @@ +#!/usr/bin/awk -f +# +# Uses MacOS' /usr/bin/sample to generate a flamegraph of a process +# +# Usage: +# +# sudo sample [pid] -file /dev/stdout | stackcollapse-sample.awk | flamegraph.pl +# +# Options: +# +# The output will show the name of the library/framework at the call-site +# with the form AppKit`NSApplication or libsystem`start_wqthread. +# +# If showing the framework or library name is not required, pass +# MODULES=0 as an argument of the sample program. +# +# The generated SVG will be written to the output stream, and can be piped +# into flamegraph.pl directly, or written to a file for conversion later. +# +# --- +# +# Copyright (c) 2017, Apple Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +BEGIN { + + # Command line options + MODULES = 1 # Allows the user to enable/disable printing of modules. + + # Internal variables + _FOUND_STACK = 0 # Found the stack traces in the output. + _LEVEL = -1 # The current level of indentation we are running. + + # The set of symbols to ignore for 'waiting' threads, for ease of use. + # This will hide waiting threads from the view, making it easier to + # see what is actually running in the sample. These may be adjusted + # as necessary or appended to if other symbols need to be filtered out. + + _IGNORE["libsystem_kernel`__psynch_cvwait"] = 1 + _IGNORE["libsystem_kernel`__select"] = 1 + _IGNORE["libsystem_kernel`__semwait_signal"] = 1 + _IGNORE["libsystem_kernel`__ulock_wait"] = 1 + _IGNORE["libsystem_kernel`__wait4"] = 1 + _IGNORE["libsystem_kernel`__workq_kernreturn"] = 1 + _IGNORE["libsystem_kernel`kevent"] = 1 + _IGNORE["libsystem_kernel`mach_msg_trap"] = 1 + _IGNORE["libsystem_kernel`read"] = 1 + _IGNORE["libsystem_kernel`semaphore_wait_trap"] = 1 + + # The same set of symbols as above, without the module name. + _IGNORE["__psynch_cvwait"] = 1 + _IGNORE["__select"] = 1 + _IGNORE["__semwait_signal"] = 1 + _IGNORE["__ulock_wait"] = 1 + _IGNORE["__wait4"] = 1 + _IGNORE["__workq_kernreturn"] = 1 + _IGNORE["kevent"] = 1 + _IGNORE["mach_msg_trap"] = 1 + _IGNORE["read"] = 1 + _IGNORE["semaphore_wait_trap"] = 1 + +} + +# This is the first line in the /usr/bin/sample output that indicates the +# samples follow subsequently. Until we see this line, the rest is ignored. + +/^Call graph/ { + _FOUND_STACK = 1 +} + +# This is found when we have reached the end of the stack output. +# Identified by the string "Total number in stack (...)". + +/^Total number/ { + _FOUND_STACK = 0 + printStack(_NEST,0) +} + +# Prints the stack from FROM to TO (where FROM > TO) +# Called when indenting back from a previous level, or at the end +# of processing to flush the last recorded sample + +function printStack(FROM,TO) { + + # We ignore certain blocking wait states, in the absence of being + # able to filter these threads from collection, otherwise + # we'll end up with many threads of equal length that represent + # the total time the sample was collected. + # + # Note that we need to collect the information to ensure that the + # timekeeping for the parental functions is appropriately adjusted + # so we just avoid printing it out when that occurs. + _PRINT_IT = !_IGNORE[_NAMES[FROM]] + + # We run through all the names, from the root to the leaf, so that + # we generate a line that flamegraph.pl will like, of the form: + # Thread1234;example`main;example`otherFn 1234 + + for(l = FROM; l>=TO; l--) { + if (_PRINT_IT) { + printf("%s", _NAMES[0]) + for(i=1; i<=l; i++) { + printf(";%s", _NAMES[i]) + } + print " " _TIMES[l] + } + + # We clean up our current state to avoid bugs. + delete _NAMES[l] + delete _TIMES[l] + } +} + +# This is where we process each line, of the form: +# 5130 Thread_8749954 +# + 5130 start_wqthread (in libsystem_pthread.dylib) ... +# + 4282 _pthread_wqthread (in libsystem_pthread.dylib) ... +# + ! 4282 __doworkq_kernreturn (in libsystem_kernel.dylib) ... +# + 848 _pthread_wqthread (in libsystem_pthread.dylib) ... +# + 848 __doworkq_kernreturn (in libsystem_kernel.dylib) ... + +_FOUND_STACK && match($0,/^ [^0-9]*[0-9]/) { + + # We maintain two counters: + # _LEVEL: the high water mark of the indentation level we have seen. + # _NEST: the current indentation level. + # + # We keep track of these two levels such that when the nesting level + # decreases, we print out the current state of where we are. + + _NEST=(RLENGTH-5)/2 + sub(/^[^0-9]*/,"") # Normalise the leading content so we start with time. + _TIME=$1 # The time recorded by 'sample', first integer value. + + # The function name is in one or two parts, depending on what kind of + # function it is. + # + # If it is a standard C or C++ function, it will be of the form: + # exampleFunction + # Example::Function + # + # If it is an Objective-C funtion, it will be of the form: + # -[NSExample function] + # +[NSExample staticFunction] + # -[NSExample function:withParameter] + # +[NSExample staticFunction:withParameter:andAnother] + + _FN1 = $2 + _FN2 = $3 + + # If it is a standard C or C++ function then the following word will + # either be blank, or the text '(in', so we jut use the first one: + + if (_FN2 == "(in" || _FN2 == "") { + _FN =_FN1 + } else { + # Otherwise we concatenate the first two parts with . + _FN = _FN1 "." _FN2 + } + + # Modules are shown with '(in libfoo.dylib)' or '(in AppKit)' + + _MODULE = "" + match($0, /\(in [^)]*\)/) + + if (RSTART > 0 && MODULES) { + + # Strip off the '(in ' (4 chars) and the final ')' char (1 char) + _MODULE = substr($0, RSTART+4, RLENGTH-5) + + # Remove the .dylib function, since it adds no value. + gsub(/\.dylib/, "", _MODULE) + + # The function name is 'module`functionName' + _FN = _MODULE "`" _FN + } + + # Now we have set up the variables, we can decide how to apply it + # If we are descending in the nesting, we don't print anything out: + # a + # ab + # abc + # + # We only print out something when we go back a level, or hit the end: + # abcd + # abe < prints out the stack up until this point, i.e. abcd + + # We store a pair of arrays, indexed by the nesting level: + # + # _TIMES - a list of the time reported to that function + # _NAMES - a list of the function names for each current stack trace + + # If we are backtracking, we need to flush the current output. + if (_NEST <= _LEVEL) { + printStack(_LEVEL,_NEST) + } + + # Record the name and time of the function where we are. + _NAMES[_NEST] = _FN + _TIMES[_NEST] = _TIME + + # We subtract the time we took from our parent so we don't double count. + if (_NEST > 0) { + _TIMES[_NEST-1] -= _TIME + } + + # Raise the high water mark of the level we have reached. + _LEVEL = _NEST +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl new file mode 100755 index 00000000000..bca4046192f --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl @@ -0,0 +1,84 @@ +#!/usr/bin/perl -w +# +# stackcollapse-stap.pl collapse multiline SystemTap stacks +# into single lines. +# +# Parses a multiline stack followed by a number on a separate line, and +# outputs a semicolon separated stack followed by a space and the number. +# If memory addresses (+0xd) are present, they are stripped, and resulting +# identical stacks are colased with their counts summed. +# +# USAGE: ./stackcollapse.pl infile > outfile +# +# Example input: +# +# 0xffffffff8103ce3b : native_safe_halt+0xb/0x10 [kernel] +# 0xffffffff8101c6a3 : default_idle+0x53/0x1d0 [kernel] +# 0xffffffff81013236 : cpu_idle+0xd6/0x120 [kernel] +# 0xffffffff815bf03e : rest_init+0x72/0x74 [kernel] +# 0xffffffff81aebbfe : start_kernel+0x3ba/0x3c5 [kernel] +# 2404 +# +# Example output: +# +# start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 2404 +# +# Input may contain many stacks as generated from SystemTap. +# +# Copyright 2011 Joyent, Inc. All rights reserved. +# Copyright 2011 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 16-Feb-2012 Brendan Gregg Created this. + +use strict; + +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} + +my @stack; + +foreach (<>) { + chomp; + + if (m/^\s*(\d+)+$/) { + remember_stack(join(";", @stack), $1); + @stack = (); + next; + } + + next if (m/^\s*$/); + + my $frame = $_; + $frame =~ s/^\s*//; + $frame =~ s/\+[^+]*$//; + $frame =~ s/.* : //; + $frame = "-" if $frame eq ""; + unshift @stack, $frame; +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + printf "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl new file mode 100755 index 00000000000..a13c1daab35 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl @@ -0,0 +1,98 @@ +#!/usr/bin/perl -w +# +# stackcollapse-vsprof.pl +# +# Parses the CSV file containing a call tree from a visual studio profiler and produces an output suitable for flamegraph.pl. +# +# USAGE: perl stackcollapse-vsprof.pl infile > outfile +# +# WORKFLOW: +# +# This example assumes you have visual studio 2015 installed. +# +# 1. Profile C++ your application using visual studio +# 2. On visual studio, choose export the call tree as csv +# 3. Generate a flamegraph: perl stackcollapse-vsprof CallTreeSummary.csv | perl flamegraph.pl > result_vsprof.svg +# +# INPUT EXAMPLE : +# +# Level,Function Name,Inclusive Samples,Exclusive Samples,Inclusive Samples %,Exclusive Samples %,Module Name, +# 1,"main","8,735",0,100.00,0.00,"an_executable.exe", +# 2,"testing::UnitTest::Run","8,735",0,100.00,0.00,"an_executable.exe", +# 3,"boost::trim_end_iter_select > >,boost::is_classifiedF>",306,16,3.50,0.18,"an_executable.exe", +# +# OUTPUT EXAMPLE : +# +# main;testing::UnitTest::Run;boost::trim_end_iter_select>>,boost::is_classifiedF> 306 + +use strict; + +sub massage_function_names; +sub parse_integer; +sub print_stack_trace; + +# data initialization +my @stack = (); +my $line_number = 0; +my $previous_samples = 0; + +my $num_args = $#ARGV + 1; +if ($num_args != 1) { + print "$ARGV[0]\n"; + print "Usage : stackcollapse-vsprof.pl > out.txt\n"; + exit; +} + +my $input_csv_file = $ARGV[0]; +my $line_parser_rx = qr{ + ^\s*(\d+?), # level in the stack + ("[^"]+" | [^,]+), # function name (beware of spaces) + ("[^"]+" | [^,]+), # number of samples (beware of locale number formatting) +}ox; + +open(my $fh, '<', $input_csv_file) or die "Can't read file '$input_csv_file' [$!]\n"; + +while (my $current_line = <$fh>){ + $line_number = $line_number + 1; + + # to discard first line which typically contains headers + next if $line_number == 1; + next if $current_line =~ /^\s*$/o; + + ($current_line =~ $line_parser_rx) or die "Error in regular expression at line $line_number : $current_line\n"; + + my $level = int $1; + my $function = massage_function_names($2); + my $samples = parse_integer($3); + my $stack_len = @stack; + + #print "[DEBUG] $line_number : $level $function $samples $stack_len\n"; + + next if not $level; + ($level <= $stack_len + 1) or die "Error in stack at line $line_number : $current_line\n"; + + if ($level <= $stack_len) { + print_stack_trace(\@stack, $previous_samples); + my $to_remove = $level - $stack_len - 1; + splice(@stack, $to_remove); + } + + $stack_len < 1000 or die "Stack overflow at line $line_number"; + push(@stack, $function); + $previous_samples = $samples; +} +print_stack_trace(\@stack, $previous_samples); + +sub massage_function_names { + return ($_[0] =~ s/\s*|^"|"$//gro); +} + +sub parse_integer { + return int ($_[0] =~ s/[., ]|^"|"$//gro); +} + +sub print_stack_trace { + my ($stack_ref, $sample) = @_; + my $stack_trace = join(";", @$stack_ref); + print "$stack_trace $sample\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl new file mode 100755 index 00000000000..e132ab08cf3 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl @@ -0,0 +1,103 @@ +#!/usr/bin/perl -w +# +# stackcollapse-vtune-mc.pl +# +# Parses the CSV file containing a call tree from Intel VTune memory-consumption profiler and produces an output suitable for flamegraph.pl. +# +# USAGE: perl stackcollapse-vtune-mc.pl [options] infile > outfile +# +# WORKFLOW: +# +# This assumes you have Intel VTune installed and on path (using Command Line) +# +# 1. Profile C++ application tachyon (example shipped with Intel VTune 2019): +# +# amplxe-cl -collect memory-consumption -r mc_tachyon -- ./tachyon +# +# 2. Export raw VTune data to csv file: +# ### for Intel VTune 2019 +# amplxe-cl -R top-down -call-stack-mode all \ +# -column="Allocations:Self","Allocation Size:Self","Module" \ +# -report-out allocations.csv -format csv \ +# -csv-delimiter comma -r mc_tachyon +# +# 3. Generate a flamegraph: +# ## Generate for allocations amount. +# perl stackcollapse-vtune-mc.pl allocations.csv > out.folded +# perl flamegraph.pl --countname=allocations out.folded > vtune_tachyon_mc.svg +# +# ## Or you can generate for allocation size in bytes. +# perl stackcollapse-vtune-mc.pl -s allocations.csv > out.folded +# perl flamegraph.pl --countname=allocations out.folded > vtune_tachyon_mc_size.svg +# +# AUTHOR: Rohith Bakkannagari +# 27-Nov-2019 UnpluggedCoder Forked from stackcollapse-vtune.pl, for memory-consumption flamegraph + +use strict; +use Getopt::Long; + +sub usage { + die < out.folded\n + --size # Accumulate allocation size in bytes instead of allocation counts.\n +NOTE : The csv file should exported by `amplxe-cl` tool with the exact -column parameter shows below. + amplxe-cl -R top-down -call-stack-mode all \ + -column="Allocations:Self","Allocation Size:Self","Module" \ + -report-out allocations.csv -format csv \ + -csv-delimiter comma -r mc_tachyon +USAGE_END +} + +# data initialization +my @stack = (); +my $rowCounter = 0; # flag for row number + +my $accSize = ''; +GetOptions ('size' => \$accSize) +or usage(); + +my $numArgs = $#ARGV + 1; +if ($numArgs != 1){ + usage(); + exit; +} + +my $inputCSVFile = $ARGV[0]; +open(my $fh, '<', $inputCSVFile) or die "Can't read file '$inputCSVFile' [$!]\n"; + +while (my $currLine = <$fh>){ + # discard warning line + next if $rowCounter == 0 && rindex($currLine, "war:", 0) == 0; + $rowCounter = $rowCounter + 1; + # to discard first row which typically contains headers + next if $rowCounter == 1; + chomp $currLine; + #VTune - sometimes the call stack information is enclosed in double quotes (?). To remove double quotes. + $currLine =~ s/\"//g; + + ### for Intel VTune 2019 + ### CSV header should be like below + ### Function Stack,Allocation Size:Self,Deallocation Size:Self,Allocations:Self,Module + $currLine =~ /(\s*)(.*?),([0-9]*?\.?[0-9]*?),([0-9]*?\.?[0-9]*?),([0-9]*?\.?[0-9]*?),(.*)/ or die "Error in regular expression on the current line $currLine\n"; + my $func = $2.'('.$6.')'; # function(module) + my $depth = length ($1); + my $allocBytes = $3; # allocation size + my $allocs = $5; # allocations + + my $tempString = ''; + $stack [$depth] = $func; + if ($accSize){ + next if $allocBytes eq ''; + foreach my $i (0 .. $depth - 1) { + $tempString = $tempString.$stack[$i].";"; + } + $tempString = $tempString.$func." $allocBytes\n"; + } else { + next if $allocs == 0; + foreach my $i (0 .. $depth - 1) { + $tempString = $tempString.$stack[$i].";"; + } + $tempString = $tempString.$func." $allocs\n"; + } + print "$tempString"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl new file mode 100644 index 00000000000..2a13e3b2d95 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl @@ -0,0 +1,97 @@ +#!/usr/bin/perl -w +# +# stackcollapse-vtune.pl +# +# Parses the CSV file containing a call tree from Intel VTune hotspots profiler and produces an output suitable for flamegraph.pl. +# +# USAGE: perl stackcollapse-vtune.pl infile > outfile +# +# WORKFLOW: +# +# This assumes you have Intel VTune installed and on path (using Command Line) +# +# 1. Profile C++ application tachyon_find_hotspots (example shipped with Intel VTune 2013): +# +# amplxe-cl -collect hotspots -r result_vtune_tachyon -- ./tachyon_find_hotspots +# +# 2. Export raw VTune data to csv file: +# +##### VTune 2013 & 2015 +# amplxe-cl -R top-down -report-out result_vtune_tachyon.csv -filter "Function Stack" -format csv -csv-delimiter comma -r result_vtune_tachyon +#### VTune 2016 +# amplxe-cl.exe -R top-down -call-stack-mode all -column="CPU Time:Self","Module" -report-output result_vtune_tachyon.csv -filter "Function Stack" -format csv -csv-delimiter comma -r result_vtune_tachyon +# +# 3. Generate a flamegraph: +# +# perl stackcollapse-vtune result_vtune_tachyon.csv | perl flamegraph.pl > result_vtune_tachyon.svg +# +# AUTHOR: Rohith Bakkannagari + +use strict; + +# data initialization +my @stack = (); +my $rowCounter = 0; #flag for row number + +my $numArgs = $#ARGV + 1; +if ($numArgs != 1) +{ +print "$ARGV[0]\n"; +print "Usage : stackcollapse-vtune.pl > out.txt\n"; +exit; +} + +my $inputCSVFile = $ARGV[0]; +my $funcOnly = ''; +my $depth = 0; +my $selfTime = 0; +my $dllName = ''; + +open(my $fh, '<', $inputCSVFile) or die "Can't read file '$inputCSVFile' [$!]\n"; + +while (my $currLine = <$fh>){ + $rowCounter = $rowCounter + 1; + # to discard first row which typically contains headers + next if $rowCounter == 1; + chomp $currLine; + + ### VTune 2013 & 2015 + #VTune - sometimes the call stack information is enclosed in double quotes (?). To remove double quotes. Not necessary for XCode instruments (MAC) + $currLine =~ s/\"//g; + $currLine =~ /(\s*)(.*),(.*),.*,([0-9]*\.?[0-9]+)/ or die "Error in regular expression on the current line\n"; + $dllName = $3; + $func = $dllName.'!'.$2; # Eg : m_lxe.dll!MathWorks::lxe::IrEngineDecorator::Apply + $depth = length ($1); + $selfTime = $4*1000; # selfTime in msec + ### VTune 2013 & 2015 + + ### VTune 2016 + # $currLine =~ /(\s*)(.*?),([0-9]*\.?[0-9]+?),(.*)/ or die "Error in regular expression on the current line $currLine\n"; + # if ($2 =~ /\"/) + # { + # $currLine =~ /(\s*)\"(.*?)\",([0-9]*\.?[0-9]+?),(.*)/ or die "Error in regular expression on the current line $currLine\n"; + # $funcOnly = $2; + # $depth = length ($1); + # $selfTime = $3*1000; # selfTime in msec + # $dllName = $4; + # } + # else + # { + # $funcOnly = $2; + # $depth = length ($1); + # $selfTime = $3*1000; # selfTime in msec + # $dllName = $4; + # } + # my $func = $dllName.'!'.$funcOnly; # Eg : m_lxe.dll!MathWorks::lxe::IrEngineDecorator::Apply + ### VTune 2016 + + my $tempString = ''; + $stack [$depth] = $func; + foreach my $i (0 .. $depth - 1) { + $tempString = $tempString.$stack[$i].";"; + } + $tempString = $tempString.$func." $selfTime\n"; + if ($selfTime != 0){ + print "$tempString"; + } +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl new file mode 100755 index 00000000000..4d1d58434a0 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl @@ -0,0 +1,69 @@ +#!/usr/bin/perl -ws +# +# stackcollapse-wcp Collapse wallClockProfiler backtraces +# +# Parse a list of GDB backtraces as generated by https://github.com/jasonrohrer/wallClockProfiler +# +# Copyright 2014 Gabriel Corona. All rights reserved. +# Portions Copyright 2020 Ștefan Talpalaru +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +use strict; + +my $current = ""; +my $start_processing = 0; +my $samples = 0; +my %stacks; + +while(<>) { + s/^\s+|\s+$//g; + + if (m/^Full stacks/) { + $start_processing = 1; + next; + } + + if (not $start_processing) { + next; + } + + if(m/^\d+\.\d+% =+ \((\d+) samples\)/) { + # 99.791% ===================================== (17194 samples) + $samples = $1; + next; + } elsif (m/^\d+: (.*)$/) { + # 1: poll__YNjd8fE6xG8CRNwfLnrx0g_2 (at /mnt/sde1/storage/nim-beacon-chain-clean/vendor/nim-chronos/chronos/asyncloop.nim:343) + my $function = $1; + if ($current eq "") { + $current = $function; + } else { + $current = $function . ";" . $current; + } + } elsif (m/^$/ and $current ne "") { + $stacks{$current} += $samples; + $current = ""; + } +} + +foreach my $k (sort { $a cmp $b } keys %stacks) { + print "$k $stacks{$k}\n"; +} + diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php b/benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php new file mode 100755 index 00000000000..52cc3d65a0c --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php @@ -0,0 +1,197 @@ +#!/usr/bin/php + outfile + -h --help Show this message + -t Weight stack counts by duration using the time index in the trace (default) + -c Invocation counts only. Simply count stacks in the trace and sum duplicates, don't weight by duration. + +Example input: +For more info on xdebug and generating traces see +https://xdebug.org/docs/execution_trace. + +Version: 2.0.0RC4-dev +TRACE START [2007-05-06 18:29:01] +1 0 0 0.010870 114112 {main} 1 ../trace.php 0 +2 1 0 0.032009 114272 str_split 0 ../trace.php 8 +2 1 1 0.032073 116632 +2 2 0 0.033505 117424 ret_ord 1 ../trace.php 10 +3 3 0 0.033531 117584 ord 0 ../trace.php 5 +3 3 1 0.033551 117584 +... +TRACE END [2007-05-06 18:29:01] + +Example output: + +- c +{main};str_split 1 +{main};ret_ord;ord 6 + +-t +{main} 23381 +{main};str_split 64 +{main};ret_ord 215 +{main};ret_ord;ord 106 + +EOT; + + exit($exit); +} + +function collapseStack(array $stack, string $func_name_key): string { + return implode(';', array_column($stack, $func_name_key)); +} + +function addCurrentStackToStacks(array $stack, float $dur, array &$stacks) { + $collapsed = implode(';', $stack); + $duration = SCALE_FACTOR * $dur; + + if (array_key_exists($collapsed, $stacks)) { + $stacks[$collapsed] += $duration; + } else { + $stacks[$collapsed] = $duration; + } +} + +function isEOTrace(string $l) { + $pattern = "/^(\\t|TRACE END)/"; + return preg_match($pattern, $l); +} + +$filename = $argv[$optind] ?? null; +if ($filename === null) { + usage(1); +} + +$do_time = !isset($args['c']); + +// First make sure our file is consistently formatted with only one \t delimiting each field +$out = []; +$retval = null; +exec("sed -in 's/\t\+/\t/g' " . escapeshellarg($filename), $out, $retval); +if ($retval !== 0) { + usage(1); +} + +$handle = fopen($filename, 'r'); + +if ($handle === false) { + echo "Unable to open $filename \n\n"; + usage(1); +} + +// Loop till we find TRACE START +while ($l = fgets($handle)) { + if (strpos($l, "TRACE START") === 0) { + break; + } +} + +const SCALE_FACTOR = 1000000; +$stacks = []; +$current_stack = []; +$was_exit = false; +$prev_start_time = 0; + +if ($do_time) { + // Weight counts by duration + // Xdebug trace time indices have 6 sigfigs of precision + // We have a perfect trace, but let's instead pretend that + // this was collected by sampling at 10^6 Hz + // then each millionth of a second this stack took to execute is 1 count + while ($l = fgets($handle)) { + if (isEOTrace($l)) { + break; + } + + $parts = explode("\t", $l); + list($level, $fn_no, $is_exit, $time) = $parts; + + if ($is_exit) { + if (empty($current_stack)) { + echo "[WARNING] Found function exit without corresponding entrance. Discarding line. Check your input.\n"; + continue; + } + + addCurrentStackToStacks($current_stack, $time - $prev_start_time, $stacks); + array_pop($current_stack); + } else { + $func_name = $parts[5]; + + if (!empty($current_stack)) { + addCurrentStackToStacks($current_stack, $time - $prev_start_time, $stacks); + } + + $current_stack[] = $func_name; + } + $prev_start_time = $time; + } +} else { + // Counts only + while ($l = fgets($handle)) { + if (isEOTrace($l)) { + break; + } + + $parts = explode("\t", $l); + list($level, $fn_no, $is_exit) = $parts; + + if ($is_exit === "1") { + if (!$was_exit) { + $collapsed = implode(";", $current_stack); + if (array_key_exists($collapsed, $stacks)) { + $stacks[$collapsed]++; + } else { + $stacks[$collapsed] = 1; + } + } + + array_pop($current_stack); + $was_exit = true; + } else { + $func_name = $parts[5]; + $current_stack[] = $func_name; + $was_exit = false; + } + } +} + +foreach ($stacks as $stack => $count) { + echo "$stack $count\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/stackcollapse.pl b/benchmarks/cli/_script/flamegraph/stackcollapse.pl new file mode 100755 index 00000000000..1e00c521368 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/stackcollapse.pl @@ -0,0 +1,109 @@ +#!/usr/bin/perl -w +# +# stackcollapse.pl collapse multiline stacks into single lines. +# +# Parses a multiline stack followed by a number on a separate line, and +# outputs a semicolon separated stack followed by a space and the number. +# If memory addresses (+0xd) are present, they are stripped, and resulting +# identical stacks are colased with their counts summed. +# +# USAGE: ./stackcollapse.pl infile > outfile +# +# Example input: +# +# unix`i86_mwait+0xd +# unix`cpu_idle_mwait+0xf1 +# unix`idle+0x114 +# unix`thread_start+0x8 +# 1641 +# +# Example output: +# +# unix`thread_start;unix`idle;unix`cpu_idle_mwait;unix`i86_mwait 1641 +# +# Input may contain many stacks, and can be generated using DTrace. The +# first few lines of input are skipped (see $headerlines). +# +# Copyright 2011 Joyent, Inc. All rights reserved. +# Copyright 2011 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 14-Aug-2011 Brendan Gregg Created this. + +use strict; + +my $headerlines = 3; # number of input lines to skip +my $includeoffset = 0; # include function offset (except leafs) +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} + +my $nr = 0; +my @stack; + +foreach (<>) { + next if $nr++ < $headerlines; + chomp; + + if (m/^\s*(\d+)+$/) { + my $count = $1; + my $joined = join(";", @stack); + + # trim leaf offset if these were retained: + $joined =~ s/\+[^+]*$// if $includeoffset; + + remember_stack($joined, $count); + @stack = (); + next; + } + + next if (m/^\s*$/); + + my $frame = $_; + $frame =~ s/^\s*//; + $frame =~ s/\+[^+]*$// unless $includeoffset; + + # Remove arguments from C++ function names: + $frame =~ s/(::.*)[(<].*/$1/; + + $frame = "-" if $frame eq ""; + + my @inline; + for (split /\->/, $frame) { + my $func = $_; + + # Strip out L and ; included in java stacks + $func =~ tr/\;/:/; + $func =~ s/^L//; + $func .= "_[i]" if scalar(@inline) > 0; #inlined + + push @inline, $func; + } + + unshift @stack, @inline; +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/benchmarks/cli/_script/flamegraph/test.sh b/benchmarks/cli/_script/flamegraph/test.sh new file mode 100755 index 00000000000..3592f351f10 --- /dev/null +++ b/benchmarks/cli/_script/flamegraph/test.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# test.sh - Check flame graph software vs test result files. +# +# This is used to detect regressions in the flame graph software. +# See record-test.sh, which refreshes these files after intended software +# changes. +# +# Currently only tests stackcollapse-perf.pl. + +set -euo pipefail +set -x +set -v + +# ToDo: add some form of --inline, and --inline --context tests. These are +# tricky since they use addr2line, whose output will vary based on the test +# system's binaries and symbol tables. +for opt in pid tid kernel jit all addrs; do + for testfile in test/*.txt ; do + echo testing $testfile : $opt + outfile=${testfile#*/} + outfile=test/results/${outfile%.txt}"-collapsed-${opt}.txt" + perl ./stackcollapse-perf.pl --"${opt}" "${testfile}" 2> /dev/null | diff -u - "${outfile}" + perl ./flamegraph.pl "${outfile}" > /dev/null + done +done diff --git a/benchmarks/cli/benchmark.sh b/benchmarks/cli/benchmark.sh new file mode 100755 index 00000000000..4f39a0b14e9 --- /dev/null +++ b/benchmarks/cli/benchmark.sh @@ -0,0 +1,381 @@ +#!/bin/bash + +set -eo pipefail + +# +# parse the command line +# + +usage() { echo "usage: $(basename "$0") [--cli ] [--name ] [--baseline-cli ] [--suite ] [--admin] [--json ] [--flamegraph] [--zip ] [--verbose] [--debug]"; } + +TEST_CLI="git" +TEST_CLI_NAME= +BASELINE_CLI= +SUITE= +JSON_RESULT= +FLAMEGRAPH= +ZIP_RESULT= +OUTPUT_DIR= +ADMIN= +VERBOSE= +DEBUG= +NEXT= + +for a in "$@"; do + if [ "${NEXT}" = "cli" ]; then + TEST_CLI="${a}" + NEXT= + elif [ "${NEXT}" = "name" ]; then + TEST_CLI_NAME="${a}" + NEXT= + elif [ "${NEXT}" = "baseline-cli" ]; then + BASELINE_CLI="${a}" + NEXT= + elif [ "${NEXT}" = "suite" ]; then + SUITE="${a}" + NEXT= + elif [ "${NEXT}" = "json" ]; then + JSON_RESULT="${a}" + NEXT= + elif [ "${NEXT}" = "zip" ]; then + ZIP_RESULT="${a}" + NEXT= + elif [ "${NEXT}" = "output-dir" ]; then + OUTPUT_DIR="${a}" + NEXT= + elif [ "${a}" = "c" ] || [ "${a}" = "--cli" ]; then + NEXT="cli" + elif [[ "${a}" == "-c"* ]]; then + TEST_CLI="${a/-c/}" + elif [ "${a}" = "n" ] || [ "${a}" = "--name" ]; then + NEXT="name" + elif [[ "${a}" == "-n"* ]]; then + TEST_CLI_NAME="${a/-n/}" + elif [ "${a}" = "b" ] || [ "${a}" = "--baseline-cli" ]; then + NEXT="baseline-cli" + elif [[ "${a}" == "-b"* ]]; then + BASELINE_CLI="${a/-b/}" + elif [ "${a}" = "-s" ] || [ "${a}" = "--suite" ]; then + NEXT="suite" + elif [[ "${a}" == "-s"* ]]; then + SUITE="${a/-s/}" + elif [ "${a}" == "--admin" ]; then + ADMIN=1 + elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then + VERBOSE=1 + elif [ "${a}" == "--debug" ]; then + VERBOSE=1 + DEBUG=1 + elif [ "${a}" = "-j" ] || [ "${a}" == "--json" ]; then + NEXT="json" + elif [[ "${a}" == "-j"* ]]; then + JSON_RESULT="${a/-j/}" + elif [ "${a}" = "-F" ] || [ "${a}" == "--flamegraph" ]; then + FLAMEGRAPH=1 + elif [ "${a}" = "-z" ] || [ "${a}" == "--zip" ]; then + NEXT="zip" + elif [[ "${a}" == "-z"* ]]; then + ZIP_RESULT="${a/-z/}" + elif [ "${a}" = "--output-dir" ]; then + NEXT="output-dir" + else + echo "$(basename "$0"): unknown option: ${a}" 1>&2 + usage 1>&2 + exit 1 + fi +done + +if [ "${NEXT}" != "" ]; then + usage 1>&2 + exit 1 +fi + +if [ "${OUTPUT_DIR}" = "" ]; then + OUTPUT_DIR=${OUTPUT_DIR:="$(mktemp -d)"} + CLEANUP_DIR=1 +fi + +# +# collect some information about the test environment +# + +SYSTEM_OS=$(uname -s) +if [ "${SYSTEM_OS}" = "Darwin" ]; then SYSTEM_OS="macOS"; fi + +SYSTEM_KERNEL=$(uname -v) + +fullpath() { + if [[ "$(uname -s)" == "MINGW"* && $(cygpath -u "${TEST_CLI}") == "/"* ]]; then + echo "$1" + elif [[ "${TEST_CLI}" == "/"* ]]; then + echo "$1" + else + which "$1" + fi +} + +cli_version() { + if [[ "$(uname -s)" == "MINGW"* ]]; then + $(cygpath -u "$1") --version + else + "$1" --version + fi +} + +cli_commit() { + if [[ "$(uname -s)" == "MINGW"* ]]; then + BUILD_OPTIONS=$($(cygpath -u "$1") version --build-options) + else + BUILD_OPTIONS=$("$1" version --build-options) + fi + + echo "${BUILD_OPTIONS}" | { grep '^built from commit: ' || echo "unknown"; } | sed -e 's/^built from commit: //' +} + +TEST_CLI_NAME=$(basename "${TEST_CLI}") +TEST_CLI_PATH=$(fullpath "${TEST_CLI}") +TEST_CLI_VERSION=$(cli_version "${TEST_CLI}") +TEST_CLI_COMMIT=$(cli_commit "${TEST_CLI}") + +if [ "${BASELINE_CLI}" != "" ]; then + if [[ "${BASELINE_CLI}" == "/"* ]]; then + BASELINE_CLI_PATH="${BASELINE_CLI}" + else + BASELINE_CLI_PATH=$(which "${BASELINE_CLI}") + fi + + BASELINE_CLI_NAME=$(basename "${BASELINE_CLI}") + BASELINE_CLI_PATH=$(fullpath "${BASELINE_CLI}") + BASELINE_CLI_VERSION=$(cli_version "${BASELINE_CLI}") + BASELINE_CLI_COMMIT=$(cli_commit "${BASELINE_CLI}") +fi + +# +# run the benchmarks +# + +echo "##############################################################################" +if [ "${SUITE}" != "" ]; then + SUITE_PREFIX="${SUITE/::/__}" + echo "## Running ${SUITE} benchmarks" +else + echo "## Running all benchmarks" +fi +echo "##############################################################################" +echo "" + +if [ "${BASELINE_CLI}" != "" ]; then + echo "# Baseline CLI: ${BASELINE_CLI} (${BASELINE_CLI_VERSION})" +fi +echo "# Test CLI: ${TEST_CLI} (${TEST_CLI_VERSION})" +echo "" + +BENCHMARK_DIR=${BENCHMARK_DIR:=$(dirname "$0")} +ANY_FOUND= +ANY_FAILED= + +indent() { sed "s/^/ /"; } +time_in_ms() { if [ "$(uname -s)" = "Darwin" ]; then date "+%s000"; else date "+%s%N" ; fi; } +humanize_secs() { + units=('s' 'ms' 'us' 'ns') + unit=0 + time="${1}" + + if [ "${time}" = "" ]; then + echo "" + return + fi + + # bash doesn't do floating point arithmetic. ick. + while [[ "${time}" == "0."* ]] && [ "$((unit+1))" != "${#units[*]}" ]; do + time="$(echo | awk "{ print ${time} * 1000 }")" + unit=$((unit+1)) + done + + echo "${time} ${units[$unit]}" +} + +TIME_START=$(time_in_ms) + +for TEST_PATH in "${BENCHMARK_DIR}"/*; do + TEST_FILE=$(basename "${TEST_PATH}") + + if [ ! -f "${TEST_PATH}" ] || [ ! -x "${TEST_PATH}" ]; then + continue + fi + + if [[ "${TEST_FILE}" != *"__"* ]]; then + continue + fi + + if [[ "${TEST_FILE}" != "${SUITE_PREFIX}"* ]]; then + continue + fi + + ANY_FOUND=1 + TEST_NAME="${TEST_FILE/__/::}" + + echo -n "${TEST_NAME}:" + if [ "${VERBOSE}" = "1" ]; then + echo "" + else + echo -n " " + fi + + if [ "${DEBUG}" = "1" ]; then + SHOW_OUTPUT="--show-output" + fi + + if [ "${ADMIN}" = "1" ]; then + ALLOW_ADMIN="--admin" + fi + + OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out" + ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err" + JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json" + FLAMEGRAPH_FILE="${OUTPUT_DIR}/${TEST_FILE}.svg" + + FAILED= + { + ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${ALLOW_ADMIN} ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}"; + FAILED=$? + } || true + + if [ "${FAILED}" = "2" ]; then + if [ "${VERBOSE}" != "1" ]; then + echo "skipped!" + fi + + indent < "${ERROR_FILE}" + continue + elif [ "${FAILED}" != "0" ]; then + if [ "${VERBOSE}" != "1" ]; then + echo "failed!" + fi + + indent < "${ERROR_FILE}" + ANY_FAILED=1 + continue + fi + + # in verbose mode, just print the hyperfine results; otherwise, + # pull the useful information out of its json and summarize it + if [ "${VERBOSE}" = "1" ]; then + indent < "${OUTPUT_FILE}" + else + jq -r '[ .results[0].mean, .results[0].stddev, .results[1].mean, .results[1].stddev ] | @tsv' < "${JSON_FILE}" | while IFS=$'\t' read -r one_mean one_stddev two_mean two_stddev; do + one_mean=$(humanize_secs "${one_mean}") + one_stddev=$(humanize_secs "${one_stddev}") + + if [ "${two_mean}" != "" ]; then + two_mean=$(humanize_secs "${two_mean}") + two_stddev=$(humanize_secs "${two_stddev}") + + echo -n "${one_mean} ± ${one_stddev} vs ${two_mean} ± ${two_stddev}" + else + echo -n "${one_mean} ± ${one_stddev}" + fi + done + fi + + # add our metadata to the hyperfine json result + jq ". |= { \"name\": \"${TEST_NAME}\" } + ." < "${JSON_FILE}" > "${JSON_FILE}.new" && mv "${JSON_FILE}.new" "${JSON_FILE}" + + # run with flamegraph output if requested + if [ "${FLAMEGRAPH}" ]; then + PROFILER_OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}-profiler.out" + PROFILER_ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}-profiler.err" + + if [ "${VERBOSE}" = "1" ]; then + echo " Profiling and creating flamegraph ..." + else + echo -n " -- profiling..." + fi + + RESULT= + { ${TEST_PATH} --cli "${TEST_CLI}" --profile --flamegraph "${FLAMEGRAPH_FILE}" >>"${PROFILER_OUTPUT_FILE}" 2>>"${PROFILER_ERROR_FILE}" || RESULT=$?; } + + if [ "${VERBOSE}" = "1" ]; then + indent < "${PROFILER_OUTPUT_FILE}" + indent < "${PROFILER_ERROR_FILE}" + else + # error code 2 indicates a non-fatal error creating + # the flamegraph + if [ "${RESULT}" = "" -o "${RESULT}" = "0" ]; then + echo " done." + elif [ "${RESULT}" = "2" ]; then + echo " missing resources." + elif [ "${RESULT}" = "3" ]; then + echo " sample too small." + + indent < "${PROFILER_ERROR_FILE}" + elif [ "${RESULT}" = "4" ]; then + echo " unavailable." + else + echo " failed." + + indent < "${PROFILER_ERROR_FILE}" + ANY_FAILED=1 + fi + fi + else + echo "" + fi +done + +TIME_END=$(time_in_ms) + +if [ "$ANY_FOUND" != "1" ]; then + echo "" + echo "error: no benchmark suite \"${SUITE}\"." + echo "" + exit 1 +fi + +escape() { + echo "${1//\\/\\\\}" +} + +# combine all the individual benchmark results into a single json file +if [ "${JSON_RESULT}" != "" ]; then + if [ "${VERBOSE}" = "1" ]; then + echo "" + echo "# Writing JSON results: ${JSON_RESULT}" + fi + + SYSTEM_JSON="{ \"os\": \"${SYSTEM_OS}\", \"kernel\": \"${SYSTEM_KERNEL}\" }" + TIME_JSON="{ \"start\": ${TIME_START}, \"end\": ${TIME_END} }" + TEST_CLI_JSON="{ \"name\": \"${TEST_CLI_NAME}\", \"path\": \"$(escape "${TEST_CLI_PATH}")\", \"version\": \"${TEST_CLI_VERSION}\", \"commit\": \"${TEST_CLI_COMMIT}\" }" + BASELINE_CLI_JSON="{ \"name\": \"${BASELINE_CLI_NAME}\", \"path\": \"$(escape "${BASELINE_CLI_PATH}")\", \"version\": \"${BASELINE_CLI_VERSION}\", \"commit\": \"${BASELINE_CLI_COMMIT}\" }" + + if [ "${BASELINE_CLI}" != "" ]; then + EXECUTOR_JSON="{ \"baseline\": ${BASELINE_CLI_JSON}, \"cli\": ${TEST_CLI_JSON} }" + else + EXECUTOR_JSON="{ \"cli\": ${TEST_CLI_JSON} }" + fi + + # add our metadata to all the test results + jq -n "{ \"system\": ${SYSTEM_JSON}, \"time\": ${TIME_JSON}, \"executor\": ${EXECUTOR_JSON}, \"tests\": [inputs] }" "${OUTPUT_DIR}"/*.json > "${JSON_RESULT}" +fi + +# combine all the data into a zip if requested +if [ "${ZIP_RESULT}" != "" ]; then + if [ "${VERBOSE}" = "1" ]; then + if [ "${JSON_RESULT}" = "" ]; then echo ""; fi + echo "# Writing ZIP results: ${ZIP_RESULT}" + fi + + zip -jr "${ZIP_RESULT}" "${OUTPUT_DIR}" >/dev/null +fi + +if [ "$CLEANUP_DIR" = "1" ]; then + rm -f "${OUTPUT_DIR}"/*.out + rm -f "${OUTPUT_DIR}"/*.err + rm -f "${OUTPUT_DIR}"/*.json + rm -f "${OUTPUT_DIR}"/*.svg + rmdir "${OUTPUT_DIR}" +fi + +if [ "$ANY_FAILED" = "1" ]; then + exit 1 +fi diff --git a/benchmarks/cli/benchmark_helpers.sh b/benchmarks/cli/benchmark_helpers.sh new file mode 100644 index 00000000000..f78aabc09f8 --- /dev/null +++ b/benchmarks/cli/benchmark_helpers.sh @@ -0,0 +1,589 @@ +# variables that benchmark tests can set +# + +set -eo pipefail + +# +# command-line parsing +# + +usage() { echo "usage: $(basename "$0") [--cli ] [--baseline-cli ] [--admin] [--output-style