diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..a113c01c6 --- /dev/null +++ b/.clang-format @@ -0,0 +1,225 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveAssignments: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + PadOperators: false +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +AttributeMacros: + - __capability +BinPackArguments: true +BinPackParameters: true +BitFieldColonSpacing: Both +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakAfterAttributes: Never +BreakAfterJavaFieldAnnotations: false +BreakArrays: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Attach +BreakBeforeInlineASMColon: OnlyMultiline +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseBlocks: false +IndentCaseLabels: false +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertBraces: false +InsertNewlineAtEOF: false +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigits: 0 + Decimal: 0 + DecimalMinDigits: 0 + Hex: 0 + HexMinDigits: 0 +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 4 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: BinPack +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +PPIndentWidth: -1 +QualifierAlignment: Leave +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceAroundPointerQualifiers: Default +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseTab: Never +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE +... + diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 000000000..952c0cca8 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,24 @@ +--- +Checks: > + bugprone-*, + -bugprone-easily-swappable-parameters, + -bugprone-implicit-widening-of-multiplication-result, + -bugprone-misplaced-widening-cast, + -bugprone-narrowing-conversions, + readability-*, + -readability-avoid-unconditional-preprocessor-if, + -readability-function-cognitive-complexity, + -readability-identifier-length, + -readability-implicit-bool-conversion, + -readability-magic-numbers, + -readability-uppercase-literal-suffix, + -readability-simplify-boolean-expr, + clang-analyzer-*, + -clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling, + performance-*, + portability-*, + misc-*, + -misc-const-correctness, + -misc-non-private-member-variables-in-classes, + -misc-no-recursion, +FormatStyle: none diff --git a/.github/build.bat b/.github/build.bat new file mode 100755 index 000000000..a904405e8 --- /dev/null +++ b/.github/build.bat @@ -0,0 +1,7 @@ +@echo off + +mkdir build +cmake -Bbuild %* +cmake --build build --config Release + +if errorlevel 1 exit /b %ERRORLEVEL% \ No newline at end of file diff --git a/.github/build.sh b/.github/build.sh new file mode 100755 index 000000000..2842d7e6d --- /dev/null +++ b/.github/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +mkdir -p build +cmake -Bbuild $@ || exit 1 +cmake --build build --config Release -j4 || exit 1 diff --git a/.github/build_cuda_linux.sh b/.github/build_cuda_linux.sh new file mode 100755 index 000000000..147c21747 --- /dev/null +++ b/.github/build_cuda_linux.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# A Cuda 12.1 install script for RHEL8/Rocky8/Manylinux_2.28 + +sudo dnf install -y kernel-devel kernel-headers +sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm +sudo dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo + +# We prefer CUDA 12.1 as it's compatible with 12.2+ +sudo dnf install -y cuda-toolkit-12-1 + +exec .github/build.sh $@ -DGGML_CUDA=1 -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.1/bin/nvcc \ No newline at end of file diff --git a/.github/dockcross/dockcross-android-arm b/.github/dockcross/dockcross-android-arm new file mode 100755 index 000000000..9cb27365d --- /dev/null +++ b/.github/dockcross/dockcross-android-arm @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm:20240418-88c04a4 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/android-arm:20240418-88c04a4 image, run: +# +# docker run --rm dockcross/android-arm:20240418-88c04a4 > dockcross-android-arm-20240418-88c04a4 +# chmod +x dockcross-android-arm-20240418-88c04a4 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/.github/dockcross/dockcross-android-arm64 b/.github/dockcross/dockcross-android-arm64 new file mode 100755 index 000000000..504527544 --- /dev/null +++ b/.github/dockcross/dockcross-android-arm64 @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm64:20240418-88c04a4 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/android-arm64:20240418-88c04a4 image, run: +# +# docker run --rm dockcross/android-arm64:20240418-88c04a4 > dockcross-android-arm64-20240418-88c04a4 +# chmod +x dockcross-android-arm64-20240418-88c04a4 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/.github/dockcross/dockcross-linux-arm64-lts b/.github/dockcross/dockcross-linux-arm64-lts new file mode 100755 index 000000000..6afd72f6b --- /dev/null +++ b/.github/dockcross/dockcross-linux-arm64-lts @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64-lts:20230601-c2f5366 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/linux-arm64-lts:20230601-c2f5366 image, run: +# +# docker run --rm dockcross/linux-arm64-lts:20230601-c2f5366 > dockcross-linux-arm64-lts-20230601-c2f5366 +# chmod +x dockcross-linux-arm64-lts-20230601-c2f5366 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/.github/dockcross/dockcross-manylinux2014-x64 b/.github/dockcross/dockcross-manylinux2014-x64 new file mode 100755 index 000000000..5fc98484e --- /dev/null +++ b/.github/dockcross/dockcross-manylinux2014-x64 @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/manylinux2014-x64:20230601-c2f5366 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/manylinux2014-x64:20230601-c2f5366 image, run: +# +# docker run --rm dockcross/manylinux2014-x64:20230601-c2f5366 > dockcross-manylinux2014-x64-20230601-c2f5366 +# chmod +x dockcross-manylinux2014-x64-20230601-c2f5366 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/.github/dockcross/dockcross-manylinux_2_28-x64 b/.github/dockcross/dockcross-manylinux_2_28-x64 new file mode 100755 index 000000000..c363e9fa4 --- /dev/null +++ b/.github/dockcross/dockcross-manylinux_2_28-x64 @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/manylinux_2_28-x64:20240812-60fa1b0 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/manylinux_2_28-x64:20240812-60fa1b0 image, run: +# +# docker run --rm dockcross/manylinux_2_28-x64:20240812-60fa1b0 > dockcross-manylinux_2_28-x64-20240812-60fa1b0 +# chmod +x dockcross-manylinux_2_28-x64-20240812-60fa1b0 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/.github/dockcross/update.sh b/.github/dockcross/update.sh new file mode 100755 index 000000000..5898ac805 --- /dev/null +++ b/.github/dockcross/update.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# This script prints the commands to upgrade the docker cross compilation scripts +docker run --rm dockcross/manylinux2014-x64 > ./dockcross-manylinux2014-x64 +docker run --rm dockcross/manylinux_2_28-x64 > ./dockcross-manylinux_2_28-x64 +docker run --rm dockcross/manylinux2014-x86 > ./dockcross-manylinux2014-x86 +docker run --rm dockcross/linux-arm64-lts > ./dockcross-linux-arm64-lts +docker run --rm dockcross/android-arm > ./dockcross-android-arm +docker run --rm dockcross/android-arm64 > ./dockcross-android-arm64 +docker run --rm dockcross/android-x86 > ./dockcross-android-x86 +docker run --rm dockcross/android-x86_64 > ./dockcross-android-x86_64 +chmod +x ./dockcross-* diff --git a/.github/include/unix/jni.h b/.github/include/unix/jni.h new file mode 100644 index 000000000..c85da1bc6 --- /dev/null +++ b/.github/include/unix/jni.h @@ -0,0 +1,2001 @@ +/* + * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * We used part of Netscape's Java Runtime Interface (JRI) as the starting + * point of our design and implementation. + */ + +/****************************************************************************** + * Java Runtime Interface + * Copyright (c) 1996 Netscape Communications Corporation. All rights reserved. + *****************************************************************************/ + +#ifndef _JAVASOFT_JNI_H_ +#define _JAVASOFT_JNI_H_ + +#include +#include + +/* jni_md.h contains the machine-dependent typedefs for jbyte, jint + and jlong */ + +#include "jni_md.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * JNI Types + */ + +#ifndef JNI_TYPES_ALREADY_DEFINED_IN_JNI_MD_H + +typedef unsigned char jboolean; +typedef unsigned short jchar; +typedef short jshort; +typedef float jfloat; +typedef double jdouble; + +typedef jint jsize; + +#ifdef __cplusplus + +class _jobject {}; +class _jclass : public _jobject {}; +class _jthrowable : public _jobject {}; +class _jstring : public _jobject {}; +class _jarray : public _jobject {}; +class _jbooleanArray : public _jarray {}; +class _jbyteArray : public _jarray {}; +class _jcharArray : public _jarray {}; +class _jshortArray : public _jarray {}; +class _jintArray : public _jarray {}; +class _jlongArray : public _jarray {}; +class _jfloatArray : public _jarray {}; +class _jdoubleArray : public _jarray {}; +class _jobjectArray : public _jarray {}; + +typedef _jobject *jobject; +typedef _jclass *jclass; +typedef _jthrowable *jthrowable; +typedef _jstring *jstring; +typedef _jarray *jarray; +typedef _jbooleanArray *jbooleanArray; +typedef _jbyteArray *jbyteArray; +typedef _jcharArray *jcharArray; +typedef _jshortArray *jshortArray; +typedef _jintArray *jintArray; +typedef _jlongArray *jlongArray; +typedef _jfloatArray *jfloatArray; +typedef _jdoubleArray *jdoubleArray; +typedef _jobjectArray *jobjectArray; + +#else + +struct _jobject; + +typedef struct _jobject *jobject; +typedef jobject jclass; +typedef jobject jthrowable; +typedef jobject jstring; +typedef jobject jarray; +typedef jarray jbooleanArray; +typedef jarray jbyteArray; +typedef jarray jcharArray; +typedef jarray jshortArray; +typedef jarray jintArray; +typedef jarray jlongArray; +typedef jarray jfloatArray; +typedef jarray jdoubleArray; +typedef jarray jobjectArray; + +#endif + +typedef jobject jweak; + +typedef union jvalue { + jboolean z; + jbyte b; + jchar c; + jshort s; + jint i; + jlong j; + jfloat f; + jdouble d; + jobject l; +} jvalue; + +struct _jfieldID; +typedef struct _jfieldID *jfieldID; + +struct _jmethodID; +typedef struct _jmethodID *jmethodID; + +/* Return values from jobjectRefType */ +typedef enum _jobjectType { + JNIInvalidRefType = 0, + JNILocalRefType = 1, + JNIGlobalRefType = 2, + JNIWeakGlobalRefType = 3 +} jobjectRefType; + + +#endif /* JNI_TYPES_ALREADY_DEFINED_IN_JNI_MD_H */ + +/* + * jboolean constants + */ + +#define JNI_FALSE 0 +#define JNI_TRUE 1 + +/* + * possible return values for JNI functions. + */ + +#define JNI_OK 0 /* success */ +#define JNI_ERR (-1) /* unknown error */ +#define JNI_EDETACHED (-2) /* thread detached from the VM */ +#define JNI_EVERSION (-3) /* JNI version error */ +#define JNI_ENOMEM (-4) /* not enough memory */ +#define JNI_EEXIST (-5) /* VM already created */ +#define JNI_EINVAL (-6) /* invalid arguments */ + +/* + * used in ReleaseScalarArrayElements + */ + +#define JNI_COMMIT 1 +#define JNI_ABORT 2 + +/* + * used in RegisterNatives to describe native method name, signature, + * and function pointer. + */ + +typedef struct { + char *name; + char *signature; + void *fnPtr; +} JNINativeMethod; + +/* + * JNI Native Method Interface. + */ + +struct JNINativeInterface_; + +struct JNIEnv_; + +#ifdef __cplusplus +typedef JNIEnv_ JNIEnv; +#else +typedef const struct JNINativeInterface_ *JNIEnv; +#endif + +/* + * JNI Invocation Interface. + */ + +struct JNIInvokeInterface_; + +struct JavaVM_; + +#ifdef __cplusplus +typedef JavaVM_ JavaVM; +#else +typedef const struct JNIInvokeInterface_ *JavaVM; +#endif + +struct JNINativeInterface_ { + void *reserved0; + void *reserved1; + void *reserved2; + + void *reserved3; + jint (JNICALL *GetVersion)(JNIEnv *env); + + jclass (JNICALL *DefineClass) + (JNIEnv *env, const char *name, jobject loader, const jbyte *buf, + jsize len); + jclass (JNICALL *FindClass) + (JNIEnv *env, const char *name); + + jmethodID (JNICALL *FromReflectedMethod) + (JNIEnv *env, jobject method); + jfieldID (JNICALL *FromReflectedField) + (JNIEnv *env, jobject field); + + jobject (JNICALL *ToReflectedMethod) + (JNIEnv *env, jclass cls, jmethodID methodID, jboolean isStatic); + + jclass (JNICALL *GetSuperclass) + (JNIEnv *env, jclass sub); + jboolean (JNICALL *IsAssignableFrom) + (JNIEnv *env, jclass sub, jclass sup); + + jobject (JNICALL *ToReflectedField) + (JNIEnv *env, jclass cls, jfieldID fieldID, jboolean isStatic); + + jint (JNICALL *Throw) + (JNIEnv *env, jthrowable obj); + jint (JNICALL *ThrowNew) + (JNIEnv *env, jclass clazz, const char *msg); + jthrowable (JNICALL *ExceptionOccurred) + (JNIEnv *env); + void (JNICALL *ExceptionDescribe) + (JNIEnv *env); + void (JNICALL *ExceptionClear) + (JNIEnv *env); + void (JNICALL *FatalError) + (JNIEnv *env, const char *msg); + + jint (JNICALL *PushLocalFrame) + (JNIEnv *env, jint capacity); + jobject (JNICALL *PopLocalFrame) + (JNIEnv *env, jobject result); + + jobject (JNICALL *NewGlobalRef) + (JNIEnv *env, jobject lobj); + void (JNICALL *DeleteGlobalRef) + (JNIEnv *env, jobject gref); + void (JNICALL *DeleteLocalRef) + (JNIEnv *env, jobject obj); + jboolean (JNICALL *IsSameObject) + (JNIEnv *env, jobject obj1, jobject obj2); + jobject (JNICALL *NewLocalRef) + (JNIEnv *env, jobject ref); + jint (JNICALL *EnsureLocalCapacity) + (JNIEnv *env, jint capacity); + + jobject (JNICALL *AllocObject) + (JNIEnv *env, jclass clazz); + jobject (JNICALL *NewObject) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jobject (JNICALL *NewObjectV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jobject (JNICALL *NewObjectA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jclass (JNICALL *GetObjectClass) + (JNIEnv *env, jobject obj); + jboolean (JNICALL *IsInstanceOf) + (JNIEnv *env, jobject obj, jclass clazz); + + jmethodID (JNICALL *GetMethodID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + + jobject (JNICALL *CallObjectMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jobject (JNICALL *CallObjectMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jobject (JNICALL *CallObjectMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue * args); + + jboolean (JNICALL *CallBooleanMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jboolean (JNICALL *CallBooleanMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jboolean (JNICALL *CallBooleanMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue * args); + + jbyte (JNICALL *CallByteMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jbyte (JNICALL *CallByteMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jbyte (JNICALL *CallByteMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jchar (JNICALL *CallCharMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jchar (JNICALL *CallCharMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jchar (JNICALL *CallCharMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jshort (JNICALL *CallShortMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jshort (JNICALL *CallShortMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jshort (JNICALL *CallShortMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jint (JNICALL *CallIntMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jint (JNICALL *CallIntMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jint (JNICALL *CallIntMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jlong (JNICALL *CallLongMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jlong (JNICALL *CallLongMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jlong (JNICALL *CallLongMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jfloat (JNICALL *CallFloatMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jfloat (JNICALL *CallFloatMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jfloat (JNICALL *CallFloatMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jdouble (JNICALL *CallDoubleMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jdouble (JNICALL *CallDoubleMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jdouble (JNICALL *CallDoubleMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + void (JNICALL *CallVoidMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + void (JNICALL *CallVoidMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + void (JNICALL *CallVoidMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue * args); + + jobject (JNICALL *CallNonvirtualObjectMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jobject (JNICALL *CallNonvirtualObjectMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jobject (JNICALL *CallNonvirtualObjectMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue * args); + + jboolean (JNICALL *CallNonvirtualBooleanMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jboolean (JNICALL *CallNonvirtualBooleanMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jboolean (JNICALL *CallNonvirtualBooleanMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue * args); + + jbyte (JNICALL *CallNonvirtualByteMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jbyte (JNICALL *CallNonvirtualByteMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jbyte (JNICALL *CallNonvirtualByteMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jchar (JNICALL *CallNonvirtualCharMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jchar (JNICALL *CallNonvirtualCharMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jchar (JNICALL *CallNonvirtualCharMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jshort (JNICALL *CallNonvirtualShortMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jshort (JNICALL *CallNonvirtualShortMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jshort (JNICALL *CallNonvirtualShortMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jint (JNICALL *CallNonvirtualIntMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jint (JNICALL *CallNonvirtualIntMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jint (JNICALL *CallNonvirtualIntMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jlong (JNICALL *CallNonvirtualLongMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jlong (JNICALL *CallNonvirtualLongMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jlong (JNICALL *CallNonvirtualLongMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jfloat (JNICALL *CallNonvirtualFloatMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jfloat (JNICALL *CallNonvirtualFloatMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jfloat (JNICALL *CallNonvirtualFloatMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jdouble (JNICALL *CallNonvirtualDoubleMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jdouble (JNICALL *CallNonvirtualDoubleMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jdouble (JNICALL *CallNonvirtualDoubleMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + void (JNICALL *CallNonvirtualVoidMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + void (JNICALL *CallNonvirtualVoidMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + void (JNICALL *CallNonvirtualVoidMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue * args); + + jfieldID (JNICALL *GetFieldID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + + jobject (JNICALL *GetObjectField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jboolean (JNICALL *GetBooleanField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jbyte (JNICALL *GetByteField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jchar (JNICALL *GetCharField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jshort (JNICALL *GetShortField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jint (JNICALL *GetIntField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jlong (JNICALL *GetLongField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jfloat (JNICALL *GetFloatField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jdouble (JNICALL *GetDoubleField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + + void (JNICALL *SetObjectField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jobject val); + void (JNICALL *SetBooleanField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jboolean val); + void (JNICALL *SetByteField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jbyte val); + void (JNICALL *SetCharField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jchar val); + void (JNICALL *SetShortField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jshort val); + void (JNICALL *SetIntField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jint val); + void (JNICALL *SetLongField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jlong val); + void (JNICALL *SetFloatField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jfloat val); + void (JNICALL *SetDoubleField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jdouble val); + + jmethodID (JNICALL *GetStaticMethodID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + + jobject (JNICALL *CallStaticObjectMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jobject (JNICALL *CallStaticObjectMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jobject (JNICALL *CallStaticObjectMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jboolean (JNICALL *CallStaticBooleanMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jboolean (JNICALL *CallStaticBooleanMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jboolean (JNICALL *CallStaticBooleanMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jbyte (JNICALL *CallStaticByteMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jbyte (JNICALL *CallStaticByteMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jbyte (JNICALL *CallStaticByteMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jchar (JNICALL *CallStaticCharMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jchar (JNICALL *CallStaticCharMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jchar (JNICALL *CallStaticCharMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jshort (JNICALL *CallStaticShortMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jshort (JNICALL *CallStaticShortMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jshort (JNICALL *CallStaticShortMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jint (JNICALL *CallStaticIntMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jint (JNICALL *CallStaticIntMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jint (JNICALL *CallStaticIntMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jlong (JNICALL *CallStaticLongMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jlong (JNICALL *CallStaticLongMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jlong (JNICALL *CallStaticLongMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jfloat (JNICALL *CallStaticFloatMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jfloat (JNICALL *CallStaticFloatMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jfloat (JNICALL *CallStaticFloatMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jdouble (JNICALL *CallStaticDoubleMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jdouble (JNICALL *CallStaticDoubleMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jdouble (JNICALL *CallStaticDoubleMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + void (JNICALL *CallStaticVoidMethod) + (JNIEnv *env, jclass cls, jmethodID methodID, ...); + void (JNICALL *CallStaticVoidMethodV) + (JNIEnv *env, jclass cls, jmethodID methodID, va_list args); + void (JNICALL *CallStaticVoidMethodA) + (JNIEnv *env, jclass cls, jmethodID methodID, const jvalue * args); + + jfieldID (JNICALL *GetStaticFieldID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + jobject (JNICALL *GetStaticObjectField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jboolean (JNICALL *GetStaticBooleanField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jbyte (JNICALL *GetStaticByteField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jchar (JNICALL *GetStaticCharField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jshort (JNICALL *GetStaticShortField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jint (JNICALL *GetStaticIntField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jlong (JNICALL *GetStaticLongField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jfloat (JNICALL *GetStaticFloatField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jdouble (JNICALL *GetStaticDoubleField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + + void (JNICALL *SetStaticObjectField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jobject value); + void (JNICALL *SetStaticBooleanField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jboolean value); + void (JNICALL *SetStaticByteField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jbyte value); + void (JNICALL *SetStaticCharField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jchar value); + void (JNICALL *SetStaticShortField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jshort value); + void (JNICALL *SetStaticIntField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jint value); + void (JNICALL *SetStaticLongField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jlong value); + void (JNICALL *SetStaticFloatField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jfloat value); + void (JNICALL *SetStaticDoubleField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jdouble value); + + jstring (JNICALL *NewString) + (JNIEnv *env, const jchar *unicode, jsize len); + jsize (JNICALL *GetStringLength) + (JNIEnv *env, jstring str); + const jchar *(JNICALL *GetStringChars) + (JNIEnv *env, jstring str, jboolean *isCopy); + void (JNICALL *ReleaseStringChars) + (JNIEnv *env, jstring str, const jchar *chars); + + jstring (JNICALL *NewStringUTF) + (JNIEnv *env, const char *utf); + jsize (JNICALL *GetStringUTFLength) + (JNIEnv *env, jstring str); + const char* (JNICALL *GetStringUTFChars) + (JNIEnv *env, jstring str, jboolean *isCopy); + void (JNICALL *ReleaseStringUTFChars) + (JNIEnv *env, jstring str, const char* chars); + + + jsize (JNICALL *GetArrayLength) + (JNIEnv *env, jarray array); + + jobjectArray (JNICALL *NewObjectArray) + (JNIEnv *env, jsize len, jclass clazz, jobject init); + jobject (JNICALL *GetObjectArrayElement) + (JNIEnv *env, jobjectArray array, jsize index); + void (JNICALL *SetObjectArrayElement) + (JNIEnv *env, jobjectArray array, jsize index, jobject val); + + jbooleanArray (JNICALL *NewBooleanArray) + (JNIEnv *env, jsize len); + jbyteArray (JNICALL *NewByteArray) + (JNIEnv *env, jsize len); + jcharArray (JNICALL *NewCharArray) + (JNIEnv *env, jsize len); + jshortArray (JNICALL *NewShortArray) + (JNIEnv *env, jsize len); + jintArray (JNICALL *NewIntArray) + (JNIEnv *env, jsize len); + jlongArray (JNICALL *NewLongArray) + (JNIEnv *env, jsize len); + jfloatArray (JNICALL *NewFloatArray) + (JNIEnv *env, jsize len); + jdoubleArray (JNICALL *NewDoubleArray) + (JNIEnv *env, jsize len); + + jboolean * (JNICALL *GetBooleanArrayElements) + (JNIEnv *env, jbooleanArray array, jboolean *isCopy); + jbyte * (JNICALL *GetByteArrayElements) + (JNIEnv *env, jbyteArray array, jboolean *isCopy); + jchar * (JNICALL *GetCharArrayElements) + (JNIEnv *env, jcharArray array, jboolean *isCopy); + jshort * (JNICALL *GetShortArrayElements) + (JNIEnv *env, jshortArray array, jboolean *isCopy); + jint * (JNICALL *GetIntArrayElements) + (JNIEnv *env, jintArray array, jboolean *isCopy); + jlong * (JNICALL *GetLongArrayElements) + (JNIEnv *env, jlongArray array, jboolean *isCopy); + jfloat * (JNICALL *GetFloatArrayElements) + (JNIEnv *env, jfloatArray array, jboolean *isCopy); + jdouble * (JNICALL *GetDoubleArrayElements) + (JNIEnv *env, jdoubleArray array, jboolean *isCopy); + + void (JNICALL *ReleaseBooleanArrayElements) + (JNIEnv *env, jbooleanArray array, jboolean *elems, jint mode); + void (JNICALL *ReleaseByteArrayElements) + (JNIEnv *env, jbyteArray array, jbyte *elems, jint mode); + void (JNICALL *ReleaseCharArrayElements) + (JNIEnv *env, jcharArray array, jchar *elems, jint mode); + void (JNICALL *ReleaseShortArrayElements) + (JNIEnv *env, jshortArray array, jshort *elems, jint mode); + void (JNICALL *ReleaseIntArrayElements) + (JNIEnv *env, jintArray array, jint *elems, jint mode); + void (JNICALL *ReleaseLongArrayElements) + (JNIEnv *env, jlongArray array, jlong *elems, jint mode); + void (JNICALL *ReleaseFloatArrayElements) + (JNIEnv *env, jfloatArray array, jfloat *elems, jint mode); + void (JNICALL *ReleaseDoubleArrayElements) + (JNIEnv *env, jdoubleArray array, jdouble *elems, jint mode); + + void (JNICALL *GetBooleanArrayRegion) + (JNIEnv *env, jbooleanArray array, jsize start, jsize l, jboolean *buf); + void (JNICALL *GetByteArrayRegion) + (JNIEnv *env, jbyteArray array, jsize start, jsize len, jbyte *buf); + void (JNICALL *GetCharArrayRegion) + (JNIEnv *env, jcharArray array, jsize start, jsize len, jchar *buf); + void (JNICALL *GetShortArrayRegion) + (JNIEnv *env, jshortArray array, jsize start, jsize len, jshort *buf); + void (JNICALL *GetIntArrayRegion) + (JNIEnv *env, jintArray array, jsize start, jsize len, jint *buf); + void (JNICALL *GetLongArrayRegion) + (JNIEnv *env, jlongArray array, jsize start, jsize len, jlong *buf); + void (JNICALL *GetFloatArrayRegion) + (JNIEnv *env, jfloatArray array, jsize start, jsize len, jfloat *buf); + void (JNICALL *GetDoubleArrayRegion) + (JNIEnv *env, jdoubleArray array, jsize start, jsize len, jdouble *buf); + + void (JNICALL *SetBooleanArrayRegion) + (JNIEnv *env, jbooleanArray array, jsize start, jsize l, const jboolean *buf); + void (JNICALL *SetByteArrayRegion) + (JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte *buf); + void (JNICALL *SetCharArrayRegion) + (JNIEnv *env, jcharArray array, jsize start, jsize len, const jchar *buf); + void (JNICALL *SetShortArrayRegion) + (JNIEnv *env, jshortArray array, jsize start, jsize len, const jshort *buf); + void (JNICALL *SetIntArrayRegion) + (JNIEnv *env, jintArray array, jsize start, jsize len, const jint *buf); + void (JNICALL *SetLongArrayRegion) + (JNIEnv *env, jlongArray array, jsize start, jsize len, const jlong *buf); + void (JNICALL *SetFloatArrayRegion) + (JNIEnv *env, jfloatArray array, jsize start, jsize len, const jfloat *buf); + void (JNICALL *SetDoubleArrayRegion) + (JNIEnv *env, jdoubleArray array, jsize start, jsize len, const jdouble *buf); + + jint (JNICALL *RegisterNatives) + (JNIEnv *env, jclass clazz, const JNINativeMethod *methods, + jint nMethods); + jint (JNICALL *UnregisterNatives) + (JNIEnv *env, jclass clazz); + + jint (JNICALL *MonitorEnter) + (JNIEnv *env, jobject obj); + jint (JNICALL *MonitorExit) + (JNIEnv *env, jobject obj); + + jint (JNICALL *GetJavaVM) + (JNIEnv *env, JavaVM **vm); + + void (JNICALL *GetStringRegion) + (JNIEnv *env, jstring str, jsize start, jsize len, jchar *buf); + void (JNICALL *GetStringUTFRegion) + (JNIEnv *env, jstring str, jsize start, jsize len, char *buf); + + void * (JNICALL *GetPrimitiveArrayCritical) + (JNIEnv *env, jarray array, jboolean *isCopy); + void (JNICALL *ReleasePrimitiveArrayCritical) + (JNIEnv *env, jarray array, void *carray, jint mode); + + const jchar * (JNICALL *GetStringCritical) + (JNIEnv *env, jstring string, jboolean *isCopy); + void (JNICALL *ReleaseStringCritical) + (JNIEnv *env, jstring string, const jchar *cstring); + + jweak (JNICALL *NewWeakGlobalRef) + (JNIEnv *env, jobject obj); + void (JNICALL *DeleteWeakGlobalRef) + (JNIEnv *env, jweak ref); + + jboolean (JNICALL *ExceptionCheck) + (JNIEnv *env); + + jobject (JNICALL *NewDirectByteBuffer) + (JNIEnv* env, void* address, jlong capacity); + void* (JNICALL *GetDirectBufferAddress) + (JNIEnv* env, jobject buf); + jlong (JNICALL *GetDirectBufferCapacity) + (JNIEnv* env, jobject buf); + + /* New JNI 1.6 Features */ + + jobjectRefType (JNICALL *GetObjectRefType) + (JNIEnv* env, jobject obj); + + /* Module Features */ + + jobject (JNICALL *GetModule) + (JNIEnv* env, jclass clazz); + + /* Virtual threads */ + + jboolean (JNICALL *IsVirtualThread) + (JNIEnv* env, jobject obj); +}; + +/* + * We use inlined functions for C++ so that programmers can write: + * + * env->FindClass("java/lang/String") + * + * in C++ rather than: + * + * (*env)->FindClass(env, "java/lang/String") + * + * in C. + */ + +struct JNIEnv_ { + const struct JNINativeInterface_ *functions; +#ifdef __cplusplus + + jint GetVersion() { + return functions->GetVersion(this); + } + jclass DefineClass(const char *name, jobject loader, const jbyte *buf, + jsize len) { + return functions->DefineClass(this, name, loader, buf, len); + } + jclass FindClass(const char *name) { + return functions->FindClass(this, name); + } + jmethodID FromReflectedMethod(jobject method) { + return functions->FromReflectedMethod(this,method); + } + jfieldID FromReflectedField(jobject field) { + return functions->FromReflectedField(this,field); + } + + jobject ToReflectedMethod(jclass cls, jmethodID methodID, jboolean isStatic) { + return functions->ToReflectedMethod(this, cls, methodID, isStatic); + } + + jclass GetSuperclass(jclass sub) { + return functions->GetSuperclass(this, sub); + } + jboolean IsAssignableFrom(jclass sub, jclass sup) { + return functions->IsAssignableFrom(this, sub, sup); + } + + jobject ToReflectedField(jclass cls, jfieldID fieldID, jboolean isStatic) { + return functions->ToReflectedField(this,cls,fieldID,isStatic); + } + + jint Throw(jthrowable obj) { + return functions->Throw(this, obj); + } + jint ThrowNew(jclass clazz, const char *msg) { + return functions->ThrowNew(this, clazz, msg); + } + jthrowable ExceptionOccurred() { + return functions->ExceptionOccurred(this); + } + void ExceptionDescribe() { + functions->ExceptionDescribe(this); + } + void ExceptionClear() { + functions->ExceptionClear(this); + } + void FatalError(const char *msg) { + functions->FatalError(this, msg); + } + + jint PushLocalFrame(jint capacity) { + return functions->PushLocalFrame(this,capacity); + } + jobject PopLocalFrame(jobject result) { + return functions->PopLocalFrame(this,result); + } + + jobject NewGlobalRef(jobject lobj) { + return functions->NewGlobalRef(this,lobj); + } + void DeleteGlobalRef(jobject gref) { + functions->DeleteGlobalRef(this,gref); + } + void DeleteLocalRef(jobject obj) { + functions->DeleteLocalRef(this, obj); + } + + jboolean IsSameObject(jobject obj1, jobject obj2) { + return functions->IsSameObject(this,obj1,obj2); + } + + jobject NewLocalRef(jobject ref) { + return functions->NewLocalRef(this,ref); + } + jint EnsureLocalCapacity(jint capacity) { + return functions->EnsureLocalCapacity(this,capacity); + } + + jobject AllocObject(jclass clazz) { + return functions->AllocObject(this,clazz); + } + jobject NewObject(jclass clazz, jmethodID methodID, ...) { + va_list args; + jobject result; + va_start(args, methodID); + result = functions->NewObjectV(this,clazz,methodID,args); + va_end(args); + return result; + } + jobject NewObjectV(jclass clazz, jmethodID methodID, + va_list args) { + return functions->NewObjectV(this,clazz,methodID,args); + } + jobject NewObjectA(jclass clazz, jmethodID methodID, + const jvalue *args) { + return functions->NewObjectA(this,clazz,methodID,args); + } + + jclass GetObjectClass(jobject obj) { + return functions->GetObjectClass(this,obj); + } + jboolean IsInstanceOf(jobject obj, jclass clazz) { + return functions->IsInstanceOf(this,obj,clazz); + } + + jmethodID GetMethodID(jclass clazz, const char *name, + const char *sig) { + return functions->GetMethodID(this,clazz,name,sig); + } + + jobject CallObjectMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jobject result; + va_start(args,methodID); + result = functions->CallObjectMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jobject CallObjectMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallObjectMethodV(this,obj,methodID,args); + } + jobject CallObjectMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallObjectMethodA(this,obj,methodID,args); + } + + jboolean CallBooleanMethod(jobject obj, + jmethodID methodID, ...) { + va_list args; + jboolean result; + va_start(args,methodID); + result = functions->CallBooleanMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jboolean CallBooleanMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallBooleanMethodV(this,obj,methodID,args); + } + jboolean CallBooleanMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallBooleanMethodA(this,obj,methodID, args); + } + + jbyte CallByteMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jbyte result; + va_start(args,methodID); + result = functions->CallByteMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jbyte CallByteMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallByteMethodV(this,obj,methodID,args); + } + jbyte CallByteMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallByteMethodA(this,obj,methodID,args); + } + + jchar CallCharMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jchar result; + va_start(args,methodID); + result = functions->CallCharMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jchar CallCharMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallCharMethodV(this,obj,methodID,args); + } + jchar CallCharMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallCharMethodA(this,obj,methodID,args); + } + + jshort CallShortMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jshort result; + va_start(args,methodID); + result = functions->CallShortMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jshort CallShortMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallShortMethodV(this,obj,methodID,args); + } + jshort CallShortMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallShortMethodA(this,obj,methodID,args); + } + + jint CallIntMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jint result; + va_start(args,methodID); + result = functions->CallIntMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jint CallIntMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallIntMethodV(this,obj,methodID,args); + } + jint CallIntMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallIntMethodA(this,obj,methodID,args); + } + + jlong CallLongMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jlong result; + va_start(args,methodID); + result = functions->CallLongMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jlong CallLongMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallLongMethodV(this,obj,methodID,args); + } + jlong CallLongMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallLongMethodA(this,obj,methodID,args); + } + + jfloat CallFloatMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jfloat result; + va_start(args,methodID); + result = functions->CallFloatMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jfloat CallFloatMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallFloatMethodV(this,obj,methodID,args); + } + jfloat CallFloatMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallFloatMethodA(this,obj,methodID,args); + } + + jdouble CallDoubleMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jdouble result; + va_start(args,methodID); + result = functions->CallDoubleMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jdouble CallDoubleMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallDoubleMethodV(this,obj,methodID,args); + } + jdouble CallDoubleMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallDoubleMethodA(this,obj,methodID,args); + } + + void CallVoidMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + va_start(args,methodID); + functions->CallVoidMethodV(this,obj,methodID,args); + va_end(args); + } + void CallVoidMethodV(jobject obj, jmethodID methodID, + va_list args) { + functions->CallVoidMethodV(this,obj,methodID,args); + } + void CallVoidMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + functions->CallVoidMethodA(this,obj,methodID,args); + } + + jobject CallNonvirtualObjectMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jobject result; + va_start(args,methodID); + result = functions->CallNonvirtualObjectMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jobject CallNonvirtualObjectMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualObjectMethodV(this,obj,clazz, + methodID,args); + } + jobject CallNonvirtualObjectMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualObjectMethodA(this,obj,clazz, + methodID,args); + } + + jboolean CallNonvirtualBooleanMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jboolean result; + va_start(args,methodID); + result = functions->CallNonvirtualBooleanMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jboolean CallNonvirtualBooleanMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualBooleanMethodV(this,obj,clazz, + methodID,args); + } + jboolean CallNonvirtualBooleanMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualBooleanMethodA(this,obj,clazz, + methodID, args); + } + + jbyte CallNonvirtualByteMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jbyte result; + va_start(args,methodID); + result = functions->CallNonvirtualByteMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jbyte CallNonvirtualByteMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualByteMethodV(this,obj,clazz, + methodID,args); + } + jbyte CallNonvirtualByteMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualByteMethodA(this,obj,clazz, + methodID,args); + } + + jchar CallNonvirtualCharMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jchar result; + va_start(args,methodID); + result = functions->CallNonvirtualCharMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jchar CallNonvirtualCharMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualCharMethodV(this,obj,clazz, + methodID,args); + } + jchar CallNonvirtualCharMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualCharMethodA(this,obj,clazz, + methodID,args); + } + + jshort CallNonvirtualShortMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jshort result; + va_start(args,methodID); + result = functions->CallNonvirtualShortMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jshort CallNonvirtualShortMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualShortMethodV(this,obj,clazz, + methodID,args); + } + jshort CallNonvirtualShortMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualShortMethodA(this,obj,clazz, + methodID,args); + } + + jint CallNonvirtualIntMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jint result; + va_start(args,methodID); + result = functions->CallNonvirtualIntMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jint CallNonvirtualIntMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualIntMethodV(this,obj,clazz, + methodID,args); + } + jint CallNonvirtualIntMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualIntMethodA(this,obj,clazz, + methodID,args); + } + + jlong CallNonvirtualLongMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jlong result; + va_start(args,methodID); + result = functions->CallNonvirtualLongMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jlong CallNonvirtualLongMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualLongMethodV(this,obj,clazz, + methodID,args); + } + jlong CallNonvirtualLongMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualLongMethodA(this,obj,clazz, + methodID,args); + } + + jfloat CallNonvirtualFloatMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jfloat result; + va_start(args,methodID); + result = functions->CallNonvirtualFloatMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jfloat CallNonvirtualFloatMethodV(jobject obj, jclass clazz, + jmethodID methodID, + va_list args) { + return functions->CallNonvirtualFloatMethodV(this,obj,clazz, + methodID,args); + } + jfloat CallNonvirtualFloatMethodA(jobject obj, jclass clazz, + jmethodID methodID, + const jvalue * args) { + return functions->CallNonvirtualFloatMethodA(this,obj,clazz, + methodID,args); + } + + jdouble CallNonvirtualDoubleMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jdouble result; + va_start(args,methodID); + result = functions->CallNonvirtualDoubleMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jdouble CallNonvirtualDoubleMethodV(jobject obj, jclass clazz, + jmethodID methodID, + va_list args) { + return functions->CallNonvirtualDoubleMethodV(this,obj,clazz, + methodID,args); + } + jdouble CallNonvirtualDoubleMethodA(jobject obj, jclass clazz, + jmethodID methodID, + const jvalue * args) { + return functions->CallNonvirtualDoubleMethodA(this,obj,clazz, + methodID,args); + } + + void CallNonvirtualVoidMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + va_start(args,methodID); + functions->CallNonvirtualVoidMethodV(this,obj,clazz,methodID,args); + va_end(args); + } + void CallNonvirtualVoidMethodV(jobject obj, jclass clazz, + jmethodID methodID, + va_list args) { + functions->CallNonvirtualVoidMethodV(this,obj,clazz,methodID,args); + } + void CallNonvirtualVoidMethodA(jobject obj, jclass clazz, + jmethodID methodID, + const jvalue * args) { + functions->CallNonvirtualVoidMethodA(this,obj,clazz,methodID,args); + } + + jfieldID GetFieldID(jclass clazz, const char *name, + const char *sig) { + return functions->GetFieldID(this,clazz,name,sig); + } + + jobject GetObjectField(jobject obj, jfieldID fieldID) { + return functions->GetObjectField(this,obj,fieldID); + } + jboolean GetBooleanField(jobject obj, jfieldID fieldID) { + return functions->GetBooleanField(this,obj,fieldID); + } + jbyte GetByteField(jobject obj, jfieldID fieldID) { + return functions->GetByteField(this,obj,fieldID); + } + jchar GetCharField(jobject obj, jfieldID fieldID) { + return functions->GetCharField(this,obj,fieldID); + } + jshort GetShortField(jobject obj, jfieldID fieldID) { + return functions->GetShortField(this,obj,fieldID); + } + jint GetIntField(jobject obj, jfieldID fieldID) { + return functions->GetIntField(this,obj,fieldID); + } + jlong GetLongField(jobject obj, jfieldID fieldID) { + return functions->GetLongField(this,obj,fieldID); + } + jfloat GetFloatField(jobject obj, jfieldID fieldID) { + return functions->GetFloatField(this,obj,fieldID); + } + jdouble GetDoubleField(jobject obj, jfieldID fieldID) { + return functions->GetDoubleField(this,obj,fieldID); + } + + void SetObjectField(jobject obj, jfieldID fieldID, jobject val) { + functions->SetObjectField(this,obj,fieldID,val); + } + void SetBooleanField(jobject obj, jfieldID fieldID, + jboolean val) { + functions->SetBooleanField(this,obj,fieldID,val); + } + void SetByteField(jobject obj, jfieldID fieldID, + jbyte val) { + functions->SetByteField(this,obj,fieldID,val); + } + void SetCharField(jobject obj, jfieldID fieldID, + jchar val) { + functions->SetCharField(this,obj,fieldID,val); + } + void SetShortField(jobject obj, jfieldID fieldID, + jshort val) { + functions->SetShortField(this,obj,fieldID,val); + } + void SetIntField(jobject obj, jfieldID fieldID, + jint val) { + functions->SetIntField(this,obj,fieldID,val); + } + void SetLongField(jobject obj, jfieldID fieldID, + jlong val) { + functions->SetLongField(this,obj,fieldID,val); + } + void SetFloatField(jobject obj, jfieldID fieldID, + jfloat val) { + functions->SetFloatField(this,obj,fieldID,val); + } + void SetDoubleField(jobject obj, jfieldID fieldID, + jdouble val) { + functions->SetDoubleField(this,obj,fieldID,val); + } + + jmethodID GetStaticMethodID(jclass clazz, const char *name, + const char *sig) { + return functions->GetStaticMethodID(this,clazz,name,sig); + } + + jobject CallStaticObjectMethod(jclass clazz, jmethodID methodID, + ...) { + va_list args; + jobject result; + va_start(args,methodID); + result = functions->CallStaticObjectMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jobject CallStaticObjectMethodV(jclass clazz, jmethodID methodID, + va_list args) { + return functions->CallStaticObjectMethodV(this,clazz,methodID,args); + } + jobject CallStaticObjectMethodA(jclass clazz, jmethodID methodID, + const jvalue *args) { + return functions->CallStaticObjectMethodA(this,clazz,methodID,args); + } + + jboolean CallStaticBooleanMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jboolean result; + va_start(args,methodID); + result = functions->CallStaticBooleanMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jboolean CallStaticBooleanMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticBooleanMethodV(this,clazz,methodID,args); + } + jboolean CallStaticBooleanMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticBooleanMethodA(this,clazz,methodID,args); + } + + jbyte CallStaticByteMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jbyte result; + va_start(args,methodID); + result = functions->CallStaticByteMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jbyte CallStaticByteMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticByteMethodV(this,clazz,methodID,args); + } + jbyte CallStaticByteMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticByteMethodA(this,clazz,methodID,args); + } + + jchar CallStaticCharMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jchar result; + va_start(args,methodID); + result = functions->CallStaticCharMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jchar CallStaticCharMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticCharMethodV(this,clazz,methodID,args); + } + jchar CallStaticCharMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticCharMethodA(this,clazz,methodID,args); + } + + jshort CallStaticShortMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jshort result; + va_start(args,methodID); + result = functions->CallStaticShortMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jshort CallStaticShortMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticShortMethodV(this,clazz,methodID,args); + } + jshort CallStaticShortMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticShortMethodA(this,clazz,methodID,args); + } + + jint CallStaticIntMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jint result; + va_start(args,methodID); + result = functions->CallStaticIntMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jint CallStaticIntMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticIntMethodV(this,clazz,methodID,args); + } + jint CallStaticIntMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticIntMethodA(this,clazz,methodID,args); + } + + jlong CallStaticLongMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jlong result; + va_start(args,methodID); + result = functions->CallStaticLongMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jlong CallStaticLongMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticLongMethodV(this,clazz,methodID,args); + } + jlong CallStaticLongMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticLongMethodA(this,clazz,methodID,args); + } + + jfloat CallStaticFloatMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jfloat result; + va_start(args,methodID); + result = functions->CallStaticFloatMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jfloat CallStaticFloatMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticFloatMethodV(this,clazz,methodID,args); + } + jfloat CallStaticFloatMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticFloatMethodA(this,clazz,methodID,args); + } + + jdouble CallStaticDoubleMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jdouble result; + va_start(args,methodID); + result = functions->CallStaticDoubleMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jdouble CallStaticDoubleMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticDoubleMethodV(this,clazz,methodID,args); + } + jdouble CallStaticDoubleMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticDoubleMethodA(this,clazz,methodID,args); + } + + void CallStaticVoidMethod(jclass cls, jmethodID methodID, ...) { + va_list args; + va_start(args,methodID); + functions->CallStaticVoidMethodV(this,cls,methodID,args); + va_end(args); + } + void CallStaticVoidMethodV(jclass cls, jmethodID methodID, + va_list args) { + functions->CallStaticVoidMethodV(this,cls,methodID,args); + } + void CallStaticVoidMethodA(jclass cls, jmethodID methodID, + const jvalue * args) { + functions->CallStaticVoidMethodA(this,cls,methodID,args); + } + + jfieldID GetStaticFieldID(jclass clazz, const char *name, + const char *sig) { + return functions->GetStaticFieldID(this,clazz,name,sig); + } + jobject GetStaticObjectField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticObjectField(this,clazz,fieldID); + } + jboolean GetStaticBooleanField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticBooleanField(this,clazz,fieldID); + } + jbyte GetStaticByteField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticByteField(this,clazz,fieldID); + } + jchar GetStaticCharField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticCharField(this,clazz,fieldID); + } + jshort GetStaticShortField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticShortField(this,clazz,fieldID); + } + jint GetStaticIntField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticIntField(this,clazz,fieldID); + } + jlong GetStaticLongField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticLongField(this,clazz,fieldID); + } + jfloat GetStaticFloatField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticFloatField(this,clazz,fieldID); + } + jdouble GetStaticDoubleField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticDoubleField(this,clazz,fieldID); + } + + void SetStaticObjectField(jclass clazz, jfieldID fieldID, + jobject value) { + functions->SetStaticObjectField(this,clazz,fieldID,value); + } + void SetStaticBooleanField(jclass clazz, jfieldID fieldID, + jboolean value) { + functions->SetStaticBooleanField(this,clazz,fieldID,value); + } + void SetStaticByteField(jclass clazz, jfieldID fieldID, + jbyte value) { + functions->SetStaticByteField(this,clazz,fieldID,value); + } + void SetStaticCharField(jclass clazz, jfieldID fieldID, + jchar value) { + functions->SetStaticCharField(this,clazz,fieldID,value); + } + void SetStaticShortField(jclass clazz, jfieldID fieldID, + jshort value) { + functions->SetStaticShortField(this,clazz,fieldID,value); + } + void SetStaticIntField(jclass clazz, jfieldID fieldID, + jint value) { + functions->SetStaticIntField(this,clazz,fieldID,value); + } + void SetStaticLongField(jclass clazz, jfieldID fieldID, + jlong value) { + functions->SetStaticLongField(this,clazz,fieldID,value); + } + void SetStaticFloatField(jclass clazz, jfieldID fieldID, + jfloat value) { + functions->SetStaticFloatField(this,clazz,fieldID,value); + } + void SetStaticDoubleField(jclass clazz, jfieldID fieldID, + jdouble value) { + functions->SetStaticDoubleField(this,clazz,fieldID,value); + } + + jstring NewString(const jchar *unicode, jsize len) { + return functions->NewString(this,unicode,len); + } + jsize GetStringLength(jstring str) { + return functions->GetStringLength(this,str); + } + const jchar *GetStringChars(jstring str, jboolean *isCopy) { + return functions->GetStringChars(this,str,isCopy); + } + void ReleaseStringChars(jstring str, const jchar *chars) { + functions->ReleaseStringChars(this,str,chars); + } + + jstring NewStringUTF(const char *utf) { + return functions->NewStringUTF(this,utf); + } + jsize GetStringUTFLength(jstring str) { + return functions->GetStringUTFLength(this,str); + } + const char* GetStringUTFChars(jstring str, jboolean *isCopy) { + return functions->GetStringUTFChars(this,str,isCopy); + } + void ReleaseStringUTFChars(jstring str, const char* chars) { + functions->ReleaseStringUTFChars(this,str,chars); + } + + jsize GetArrayLength(jarray array) { + return functions->GetArrayLength(this,array); + } + + jobjectArray NewObjectArray(jsize len, jclass clazz, + jobject init) { + return functions->NewObjectArray(this,len,clazz,init); + } + jobject GetObjectArrayElement(jobjectArray array, jsize index) { + return functions->GetObjectArrayElement(this,array,index); + } + void SetObjectArrayElement(jobjectArray array, jsize index, + jobject val) { + functions->SetObjectArrayElement(this,array,index,val); + } + + jbooleanArray NewBooleanArray(jsize len) { + return functions->NewBooleanArray(this,len); + } + jbyteArray NewByteArray(jsize len) { + return functions->NewByteArray(this,len); + } + jcharArray NewCharArray(jsize len) { + return functions->NewCharArray(this,len); + } + jshortArray NewShortArray(jsize len) { + return functions->NewShortArray(this,len); + } + jintArray NewIntArray(jsize len) { + return functions->NewIntArray(this,len); + } + jlongArray NewLongArray(jsize len) { + return functions->NewLongArray(this,len); + } + jfloatArray NewFloatArray(jsize len) { + return functions->NewFloatArray(this,len); + } + jdoubleArray NewDoubleArray(jsize len) { + return functions->NewDoubleArray(this,len); + } + + jboolean * GetBooleanArrayElements(jbooleanArray array, jboolean *isCopy) { + return functions->GetBooleanArrayElements(this,array,isCopy); + } + jbyte * GetByteArrayElements(jbyteArray array, jboolean *isCopy) { + return functions->GetByteArrayElements(this,array,isCopy); + } + jchar * GetCharArrayElements(jcharArray array, jboolean *isCopy) { + return functions->GetCharArrayElements(this,array,isCopy); + } + jshort * GetShortArrayElements(jshortArray array, jboolean *isCopy) { + return functions->GetShortArrayElements(this,array,isCopy); + } + jint * GetIntArrayElements(jintArray array, jboolean *isCopy) { + return functions->GetIntArrayElements(this,array,isCopy); + } + jlong * GetLongArrayElements(jlongArray array, jboolean *isCopy) { + return functions->GetLongArrayElements(this,array,isCopy); + } + jfloat * GetFloatArrayElements(jfloatArray array, jboolean *isCopy) { + return functions->GetFloatArrayElements(this,array,isCopy); + } + jdouble * GetDoubleArrayElements(jdoubleArray array, jboolean *isCopy) { + return functions->GetDoubleArrayElements(this,array,isCopy); + } + + void ReleaseBooleanArrayElements(jbooleanArray array, + jboolean *elems, + jint mode) { + functions->ReleaseBooleanArrayElements(this,array,elems,mode); + } + void ReleaseByteArrayElements(jbyteArray array, + jbyte *elems, + jint mode) { + functions->ReleaseByteArrayElements(this,array,elems,mode); + } + void ReleaseCharArrayElements(jcharArray array, + jchar *elems, + jint mode) { + functions->ReleaseCharArrayElements(this,array,elems,mode); + } + void ReleaseShortArrayElements(jshortArray array, + jshort *elems, + jint mode) { + functions->ReleaseShortArrayElements(this,array,elems,mode); + } + void ReleaseIntArrayElements(jintArray array, + jint *elems, + jint mode) { + functions->ReleaseIntArrayElements(this,array,elems,mode); + } + void ReleaseLongArrayElements(jlongArray array, + jlong *elems, + jint mode) { + functions->ReleaseLongArrayElements(this,array,elems,mode); + } + void ReleaseFloatArrayElements(jfloatArray array, + jfloat *elems, + jint mode) { + functions->ReleaseFloatArrayElements(this,array,elems,mode); + } + void ReleaseDoubleArrayElements(jdoubleArray array, + jdouble *elems, + jint mode) { + functions->ReleaseDoubleArrayElements(this,array,elems,mode); + } + + void GetBooleanArrayRegion(jbooleanArray array, + jsize start, jsize len, jboolean *buf) { + functions->GetBooleanArrayRegion(this,array,start,len,buf); + } + void GetByteArrayRegion(jbyteArray array, + jsize start, jsize len, jbyte *buf) { + functions->GetByteArrayRegion(this,array,start,len,buf); + } + void GetCharArrayRegion(jcharArray array, + jsize start, jsize len, jchar *buf) { + functions->GetCharArrayRegion(this,array,start,len,buf); + } + void GetShortArrayRegion(jshortArray array, + jsize start, jsize len, jshort *buf) { + functions->GetShortArrayRegion(this,array,start,len,buf); + } + void GetIntArrayRegion(jintArray array, + jsize start, jsize len, jint *buf) { + functions->GetIntArrayRegion(this,array,start,len,buf); + } + void GetLongArrayRegion(jlongArray array, + jsize start, jsize len, jlong *buf) { + functions->GetLongArrayRegion(this,array,start,len,buf); + } + void GetFloatArrayRegion(jfloatArray array, + jsize start, jsize len, jfloat *buf) { + functions->GetFloatArrayRegion(this,array,start,len,buf); + } + void GetDoubleArrayRegion(jdoubleArray array, + jsize start, jsize len, jdouble *buf) { + functions->GetDoubleArrayRegion(this,array,start,len,buf); + } + + void SetBooleanArrayRegion(jbooleanArray array, jsize start, jsize len, + const jboolean *buf) { + functions->SetBooleanArrayRegion(this,array,start,len,buf); + } + void SetByteArrayRegion(jbyteArray array, jsize start, jsize len, + const jbyte *buf) { + functions->SetByteArrayRegion(this,array,start,len,buf); + } + void SetCharArrayRegion(jcharArray array, jsize start, jsize len, + const jchar *buf) { + functions->SetCharArrayRegion(this,array,start,len,buf); + } + void SetShortArrayRegion(jshortArray array, jsize start, jsize len, + const jshort *buf) { + functions->SetShortArrayRegion(this,array,start,len,buf); + } + void SetIntArrayRegion(jintArray array, jsize start, jsize len, + const jint *buf) { + functions->SetIntArrayRegion(this,array,start,len,buf); + } + void SetLongArrayRegion(jlongArray array, jsize start, jsize len, + const jlong *buf) { + functions->SetLongArrayRegion(this,array,start,len,buf); + } + void SetFloatArrayRegion(jfloatArray array, jsize start, jsize len, + const jfloat *buf) { + functions->SetFloatArrayRegion(this,array,start,len,buf); + } + void SetDoubleArrayRegion(jdoubleArray array, jsize start, jsize len, + const jdouble *buf) { + functions->SetDoubleArrayRegion(this,array,start,len,buf); + } + + jint RegisterNatives(jclass clazz, const JNINativeMethod *methods, + jint nMethods) { + return functions->RegisterNatives(this,clazz,methods,nMethods); + } + jint UnregisterNatives(jclass clazz) { + return functions->UnregisterNatives(this,clazz); + } + + jint MonitorEnter(jobject obj) { + return functions->MonitorEnter(this,obj); + } + jint MonitorExit(jobject obj) { + return functions->MonitorExit(this,obj); + } + + jint GetJavaVM(JavaVM **vm) { + return functions->GetJavaVM(this,vm); + } + + void GetStringRegion(jstring str, jsize start, jsize len, jchar *buf) { + functions->GetStringRegion(this,str,start,len,buf); + } + void GetStringUTFRegion(jstring str, jsize start, jsize len, char *buf) { + functions->GetStringUTFRegion(this,str,start,len,buf); + } + + void * GetPrimitiveArrayCritical(jarray array, jboolean *isCopy) { + return functions->GetPrimitiveArrayCritical(this,array,isCopy); + } + void ReleasePrimitiveArrayCritical(jarray array, void *carray, jint mode) { + functions->ReleasePrimitiveArrayCritical(this,array,carray,mode); + } + + const jchar * GetStringCritical(jstring string, jboolean *isCopy) { + return functions->GetStringCritical(this,string,isCopy); + } + void ReleaseStringCritical(jstring string, const jchar *cstring) { + functions->ReleaseStringCritical(this,string,cstring); + } + + jweak NewWeakGlobalRef(jobject obj) { + return functions->NewWeakGlobalRef(this,obj); + } + void DeleteWeakGlobalRef(jweak ref) { + functions->DeleteWeakGlobalRef(this,ref); + } + + jboolean ExceptionCheck() { + return functions->ExceptionCheck(this); + } + + jobject NewDirectByteBuffer(void* address, jlong capacity) { + return functions->NewDirectByteBuffer(this, address, capacity); + } + void* GetDirectBufferAddress(jobject buf) { + return functions->GetDirectBufferAddress(this, buf); + } + jlong GetDirectBufferCapacity(jobject buf) { + return functions->GetDirectBufferCapacity(this, buf); + } + jobjectRefType GetObjectRefType(jobject obj) { + return functions->GetObjectRefType(this, obj); + } + + /* Module Features */ + + jobject GetModule(jclass clazz) { + return functions->GetModule(this, clazz); + } + + /* Virtual threads */ + + jboolean IsVirtualThread(jobject obj) { + return functions->IsVirtualThread(this, obj); + } + +#endif /* __cplusplus */ +}; + +/* + * optionString may be any option accepted by the JVM, or one of the + * following: + * + * -D= Set a system property. + * -verbose[:class|gc|jni] Enable verbose output, comma-separated. E.g. + * "-verbose:class" or "-verbose:gc,class" + * Standard names include: gc, class, and jni. + * All nonstandard (VM-specific) names must begin + * with "X". + * vfprintf extraInfo is a pointer to the vfprintf hook. + * exit extraInfo is a pointer to the exit hook. + * abort extraInfo is a pointer to the abort hook. + */ +typedef struct JavaVMOption { + char *optionString; + void *extraInfo; +} JavaVMOption; + +typedef struct JavaVMInitArgs { + jint version; + + jint nOptions; + JavaVMOption *options; + jboolean ignoreUnrecognized; +} JavaVMInitArgs; + +typedef struct JavaVMAttachArgs { + jint version; + + char *name; + jobject group; +} JavaVMAttachArgs; + +/* These will be VM-specific. */ + +#define JDK1_2 +#define JDK1_4 + +/* End VM-specific. */ + +struct JNIInvokeInterface_ { + void *reserved0; + void *reserved1; + void *reserved2; + + jint (JNICALL *DestroyJavaVM)(JavaVM *vm); + + jint (JNICALL *AttachCurrentThread)(JavaVM *vm, void **penv, void *args); + + jint (JNICALL *DetachCurrentThread)(JavaVM *vm); + + jint (JNICALL *GetEnv)(JavaVM *vm, void **penv, jint version); + + jint (JNICALL *AttachCurrentThreadAsDaemon)(JavaVM *vm, void **penv, void *args); +}; + +struct JavaVM_ { + const struct JNIInvokeInterface_ *functions; +#ifdef __cplusplus + + jint DestroyJavaVM() { + return functions->DestroyJavaVM(this); + } + jint AttachCurrentThread(void **penv, void *args) { + return functions->AttachCurrentThread(this, penv, args); + } + jint DetachCurrentThread() { + return functions->DetachCurrentThread(this); + } + + jint GetEnv(void **penv, jint version) { + return functions->GetEnv(this, penv, version); + } + jint AttachCurrentThreadAsDaemon(void **penv, void *args) { + return functions->AttachCurrentThreadAsDaemon(this, penv, args); + } +#endif +}; + +#ifdef _JNI_IMPLEMENTATION_ +#define _JNI_IMPORT_OR_EXPORT_ JNIEXPORT +#else +#define _JNI_IMPORT_OR_EXPORT_ JNIIMPORT +#endif +_JNI_IMPORT_OR_EXPORT_ jint JNICALL +JNI_GetDefaultJavaVMInitArgs(void *args); + +_JNI_IMPORT_OR_EXPORT_ jint JNICALL +JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args); + +_JNI_IMPORT_OR_EXPORT_ jint JNICALL +JNI_GetCreatedJavaVMs(JavaVM **, jsize, jsize *); + +/* Defined by native libraries. */ +JNIEXPORT jint JNICALL +JNI_OnLoad(JavaVM *vm, void *reserved); + +JNIEXPORT void JNICALL +JNI_OnUnload(JavaVM *vm, void *reserved); + +#define JNI_VERSION_1_1 0x00010001 +#define JNI_VERSION_1_2 0x00010002 +#define JNI_VERSION_1_4 0x00010004 +#define JNI_VERSION_1_6 0x00010006 +#define JNI_VERSION_1_8 0x00010008 +#define JNI_VERSION_9 0x00090000 +#define JNI_VERSION_10 0x000a0000 +#define JNI_VERSION_19 0x00130000 +#define JNI_VERSION_20 0x00140000 +#define JNI_VERSION_21 0x00150000 + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* !_JAVASOFT_JNI_H_ */ diff --git a/.github/include/unix/jni_md.h b/.github/include/unix/jni_md.h new file mode 100644 index 000000000..6e352038f --- /dev/null +++ b/.github/include/unix/jni_md.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef _JAVASOFT_JNI_MD_H_ +#define _JAVASOFT_JNI_MD_H_ + +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#if (defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4) && (__GNUC_MINOR__ > 2))) || __has_attribute(visibility) + #ifdef ARM + #define JNIEXPORT __attribute__((externally_visible,visibility("default"))) + #define JNIIMPORT __attribute__((externally_visible,visibility("default"))) + #else + #define JNIEXPORT __attribute__((visibility("default"))) + #define JNIIMPORT __attribute__((visibility("default"))) + #endif +#else + #define JNIEXPORT + #define JNIIMPORT +#endif + +#define JNICALL + +typedef int jint; +#ifdef _LP64 +typedef long jlong; +#else +typedef long long jlong; +#endif + +typedef signed char jbyte; + +#endif /* !_JAVASOFT_JNI_MD_H_ */ diff --git a/.github/include/windows/jni.h b/.github/include/windows/jni.h new file mode 100644 index 000000000..c85da1bc6 --- /dev/null +++ b/.github/include/windows/jni.h @@ -0,0 +1,2001 @@ +/* + * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * We used part of Netscape's Java Runtime Interface (JRI) as the starting + * point of our design and implementation. + */ + +/****************************************************************************** + * Java Runtime Interface + * Copyright (c) 1996 Netscape Communications Corporation. All rights reserved. + *****************************************************************************/ + +#ifndef _JAVASOFT_JNI_H_ +#define _JAVASOFT_JNI_H_ + +#include +#include + +/* jni_md.h contains the machine-dependent typedefs for jbyte, jint + and jlong */ + +#include "jni_md.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * JNI Types + */ + +#ifndef JNI_TYPES_ALREADY_DEFINED_IN_JNI_MD_H + +typedef unsigned char jboolean; +typedef unsigned short jchar; +typedef short jshort; +typedef float jfloat; +typedef double jdouble; + +typedef jint jsize; + +#ifdef __cplusplus + +class _jobject {}; +class _jclass : public _jobject {}; +class _jthrowable : public _jobject {}; +class _jstring : public _jobject {}; +class _jarray : public _jobject {}; +class _jbooleanArray : public _jarray {}; +class _jbyteArray : public _jarray {}; +class _jcharArray : public _jarray {}; +class _jshortArray : public _jarray {}; +class _jintArray : public _jarray {}; +class _jlongArray : public _jarray {}; +class _jfloatArray : public _jarray {}; +class _jdoubleArray : public _jarray {}; +class _jobjectArray : public _jarray {}; + +typedef _jobject *jobject; +typedef _jclass *jclass; +typedef _jthrowable *jthrowable; +typedef _jstring *jstring; +typedef _jarray *jarray; +typedef _jbooleanArray *jbooleanArray; +typedef _jbyteArray *jbyteArray; +typedef _jcharArray *jcharArray; +typedef _jshortArray *jshortArray; +typedef _jintArray *jintArray; +typedef _jlongArray *jlongArray; +typedef _jfloatArray *jfloatArray; +typedef _jdoubleArray *jdoubleArray; +typedef _jobjectArray *jobjectArray; + +#else + +struct _jobject; + +typedef struct _jobject *jobject; +typedef jobject jclass; +typedef jobject jthrowable; +typedef jobject jstring; +typedef jobject jarray; +typedef jarray jbooleanArray; +typedef jarray jbyteArray; +typedef jarray jcharArray; +typedef jarray jshortArray; +typedef jarray jintArray; +typedef jarray jlongArray; +typedef jarray jfloatArray; +typedef jarray jdoubleArray; +typedef jarray jobjectArray; + +#endif + +typedef jobject jweak; + +typedef union jvalue { + jboolean z; + jbyte b; + jchar c; + jshort s; + jint i; + jlong j; + jfloat f; + jdouble d; + jobject l; +} jvalue; + +struct _jfieldID; +typedef struct _jfieldID *jfieldID; + +struct _jmethodID; +typedef struct _jmethodID *jmethodID; + +/* Return values from jobjectRefType */ +typedef enum _jobjectType { + JNIInvalidRefType = 0, + JNILocalRefType = 1, + JNIGlobalRefType = 2, + JNIWeakGlobalRefType = 3 +} jobjectRefType; + + +#endif /* JNI_TYPES_ALREADY_DEFINED_IN_JNI_MD_H */ + +/* + * jboolean constants + */ + +#define JNI_FALSE 0 +#define JNI_TRUE 1 + +/* + * possible return values for JNI functions. + */ + +#define JNI_OK 0 /* success */ +#define JNI_ERR (-1) /* unknown error */ +#define JNI_EDETACHED (-2) /* thread detached from the VM */ +#define JNI_EVERSION (-3) /* JNI version error */ +#define JNI_ENOMEM (-4) /* not enough memory */ +#define JNI_EEXIST (-5) /* VM already created */ +#define JNI_EINVAL (-6) /* invalid arguments */ + +/* + * used in ReleaseScalarArrayElements + */ + +#define JNI_COMMIT 1 +#define JNI_ABORT 2 + +/* + * used in RegisterNatives to describe native method name, signature, + * and function pointer. + */ + +typedef struct { + char *name; + char *signature; + void *fnPtr; +} JNINativeMethod; + +/* + * JNI Native Method Interface. + */ + +struct JNINativeInterface_; + +struct JNIEnv_; + +#ifdef __cplusplus +typedef JNIEnv_ JNIEnv; +#else +typedef const struct JNINativeInterface_ *JNIEnv; +#endif + +/* + * JNI Invocation Interface. + */ + +struct JNIInvokeInterface_; + +struct JavaVM_; + +#ifdef __cplusplus +typedef JavaVM_ JavaVM; +#else +typedef const struct JNIInvokeInterface_ *JavaVM; +#endif + +struct JNINativeInterface_ { + void *reserved0; + void *reserved1; + void *reserved2; + + void *reserved3; + jint (JNICALL *GetVersion)(JNIEnv *env); + + jclass (JNICALL *DefineClass) + (JNIEnv *env, const char *name, jobject loader, const jbyte *buf, + jsize len); + jclass (JNICALL *FindClass) + (JNIEnv *env, const char *name); + + jmethodID (JNICALL *FromReflectedMethod) + (JNIEnv *env, jobject method); + jfieldID (JNICALL *FromReflectedField) + (JNIEnv *env, jobject field); + + jobject (JNICALL *ToReflectedMethod) + (JNIEnv *env, jclass cls, jmethodID methodID, jboolean isStatic); + + jclass (JNICALL *GetSuperclass) + (JNIEnv *env, jclass sub); + jboolean (JNICALL *IsAssignableFrom) + (JNIEnv *env, jclass sub, jclass sup); + + jobject (JNICALL *ToReflectedField) + (JNIEnv *env, jclass cls, jfieldID fieldID, jboolean isStatic); + + jint (JNICALL *Throw) + (JNIEnv *env, jthrowable obj); + jint (JNICALL *ThrowNew) + (JNIEnv *env, jclass clazz, const char *msg); + jthrowable (JNICALL *ExceptionOccurred) + (JNIEnv *env); + void (JNICALL *ExceptionDescribe) + (JNIEnv *env); + void (JNICALL *ExceptionClear) + (JNIEnv *env); + void (JNICALL *FatalError) + (JNIEnv *env, const char *msg); + + jint (JNICALL *PushLocalFrame) + (JNIEnv *env, jint capacity); + jobject (JNICALL *PopLocalFrame) + (JNIEnv *env, jobject result); + + jobject (JNICALL *NewGlobalRef) + (JNIEnv *env, jobject lobj); + void (JNICALL *DeleteGlobalRef) + (JNIEnv *env, jobject gref); + void (JNICALL *DeleteLocalRef) + (JNIEnv *env, jobject obj); + jboolean (JNICALL *IsSameObject) + (JNIEnv *env, jobject obj1, jobject obj2); + jobject (JNICALL *NewLocalRef) + (JNIEnv *env, jobject ref); + jint (JNICALL *EnsureLocalCapacity) + (JNIEnv *env, jint capacity); + + jobject (JNICALL *AllocObject) + (JNIEnv *env, jclass clazz); + jobject (JNICALL *NewObject) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jobject (JNICALL *NewObjectV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jobject (JNICALL *NewObjectA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jclass (JNICALL *GetObjectClass) + (JNIEnv *env, jobject obj); + jboolean (JNICALL *IsInstanceOf) + (JNIEnv *env, jobject obj, jclass clazz); + + jmethodID (JNICALL *GetMethodID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + + jobject (JNICALL *CallObjectMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jobject (JNICALL *CallObjectMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jobject (JNICALL *CallObjectMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue * args); + + jboolean (JNICALL *CallBooleanMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jboolean (JNICALL *CallBooleanMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jboolean (JNICALL *CallBooleanMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue * args); + + jbyte (JNICALL *CallByteMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jbyte (JNICALL *CallByteMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jbyte (JNICALL *CallByteMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jchar (JNICALL *CallCharMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jchar (JNICALL *CallCharMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jchar (JNICALL *CallCharMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jshort (JNICALL *CallShortMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jshort (JNICALL *CallShortMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jshort (JNICALL *CallShortMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jint (JNICALL *CallIntMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jint (JNICALL *CallIntMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jint (JNICALL *CallIntMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jlong (JNICALL *CallLongMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jlong (JNICALL *CallLongMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jlong (JNICALL *CallLongMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jfloat (JNICALL *CallFloatMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jfloat (JNICALL *CallFloatMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jfloat (JNICALL *CallFloatMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + jdouble (JNICALL *CallDoubleMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + jdouble (JNICALL *CallDoubleMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + jdouble (JNICALL *CallDoubleMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args); + + void (JNICALL *CallVoidMethod) + (JNIEnv *env, jobject obj, jmethodID methodID, ...); + void (JNICALL *CallVoidMethodV) + (JNIEnv *env, jobject obj, jmethodID methodID, va_list args); + void (JNICALL *CallVoidMethodA) + (JNIEnv *env, jobject obj, jmethodID methodID, const jvalue * args); + + jobject (JNICALL *CallNonvirtualObjectMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jobject (JNICALL *CallNonvirtualObjectMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jobject (JNICALL *CallNonvirtualObjectMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue * args); + + jboolean (JNICALL *CallNonvirtualBooleanMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jboolean (JNICALL *CallNonvirtualBooleanMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jboolean (JNICALL *CallNonvirtualBooleanMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue * args); + + jbyte (JNICALL *CallNonvirtualByteMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jbyte (JNICALL *CallNonvirtualByteMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jbyte (JNICALL *CallNonvirtualByteMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jchar (JNICALL *CallNonvirtualCharMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jchar (JNICALL *CallNonvirtualCharMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jchar (JNICALL *CallNonvirtualCharMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jshort (JNICALL *CallNonvirtualShortMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jshort (JNICALL *CallNonvirtualShortMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jshort (JNICALL *CallNonvirtualShortMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jint (JNICALL *CallNonvirtualIntMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jint (JNICALL *CallNonvirtualIntMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jint (JNICALL *CallNonvirtualIntMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jlong (JNICALL *CallNonvirtualLongMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jlong (JNICALL *CallNonvirtualLongMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jlong (JNICALL *CallNonvirtualLongMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jfloat (JNICALL *CallNonvirtualFloatMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jfloat (JNICALL *CallNonvirtualFloatMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jfloat (JNICALL *CallNonvirtualFloatMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + jdouble (JNICALL *CallNonvirtualDoubleMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + jdouble (JNICALL *CallNonvirtualDoubleMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + jdouble (JNICALL *CallNonvirtualDoubleMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue *args); + + void (JNICALL *CallNonvirtualVoidMethod) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, ...); + void (JNICALL *CallNonvirtualVoidMethodV) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + va_list args); + void (JNICALL *CallNonvirtualVoidMethodA) + (JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, + const jvalue * args); + + jfieldID (JNICALL *GetFieldID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + + jobject (JNICALL *GetObjectField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jboolean (JNICALL *GetBooleanField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jbyte (JNICALL *GetByteField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jchar (JNICALL *GetCharField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jshort (JNICALL *GetShortField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jint (JNICALL *GetIntField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jlong (JNICALL *GetLongField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jfloat (JNICALL *GetFloatField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + jdouble (JNICALL *GetDoubleField) + (JNIEnv *env, jobject obj, jfieldID fieldID); + + void (JNICALL *SetObjectField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jobject val); + void (JNICALL *SetBooleanField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jboolean val); + void (JNICALL *SetByteField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jbyte val); + void (JNICALL *SetCharField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jchar val); + void (JNICALL *SetShortField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jshort val); + void (JNICALL *SetIntField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jint val); + void (JNICALL *SetLongField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jlong val); + void (JNICALL *SetFloatField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jfloat val); + void (JNICALL *SetDoubleField) + (JNIEnv *env, jobject obj, jfieldID fieldID, jdouble val); + + jmethodID (JNICALL *GetStaticMethodID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + + jobject (JNICALL *CallStaticObjectMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jobject (JNICALL *CallStaticObjectMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jobject (JNICALL *CallStaticObjectMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jboolean (JNICALL *CallStaticBooleanMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jboolean (JNICALL *CallStaticBooleanMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jboolean (JNICALL *CallStaticBooleanMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jbyte (JNICALL *CallStaticByteMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jbyte (JNICALL *CallStaticByteMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jbyte (JNICALL *CallStaticByteMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jchar (JNICALL *CallStaticCharMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jchar (JNICALL *CallStaticCharMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jchar (JNICALL *CallStaticCharMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jshort (JNICALL *CallStaticShortMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jshort (JNICALL *CallStaticShortMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jshort (JNICALL *CallStaticShortMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jint (JNICALL *CallStaticIntMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jint (JNICALL *CallStaticIntMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jint (JNICALL *CallStaticIntMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jlong (JNICALL *CallStaticLongMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jlong (JNICALL *CallStaticLongMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jlong (JNICALL *CallStaticLongMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jfloat (JNICALL *CallStaticFloatMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jfloat (JNICALL *CallStaticFloatMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jfloat (JNICALL *CallStaticFloatMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + jdouble (JNICALL *CallStaticDoubleMethod) + (JNIEnv *env, jclass clazz, jmethodID methodID, ...); + jdouble (JNICALL *CallStaticDoubleMethodV) + (JNIEnv *env, jclass clazz, jmethodID methodID, va_list args); + jdouble (JNICALL *CallStaticDoubleMethodA) + (JNIEnv *env, jclass clazz, jmethodID methodID, const jvalue *args); + + void (JNICALL *CallStaticVoidMethod) + (JNIEnv *env, jclass cls, jmethodID methodID, ...); + void (JNICALL *CallStaticVoidMethodV) + (JNIEnv *env, jclass cls, jmethodID methodID, va_list args); + void (JNICALL *CallStaticVoidMethodA) + (JNIEnv *env, jclass cls, jmethodID methodID, const jvalue * args); + + jfieldID (JNICALL *GetStaticFieldID) + (JNIEnv *env, jclass clazz, const char *name, const char *sig); + jobject (JNICALL *GetStaticObjectField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jboolean (JNICALL *GetStaticBooleanField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jbyte (JNICALL *GetStaticByteField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jchar (JNICALL *GetStaticCharField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jshort (JNICALL *GetStaticShortField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jint (JNICALL *GetStaticIntField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jlong (JNICALL *GetStaticLongField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jfloat (JNICALL *GetStaticFloatField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + jdouble (JNICALL *GetStaticDoubleField) + (JNIEnv *env, jclass clazz, jfieldID fieldID); + + void (JNICALL *SetStaticObjectField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jobject value); + void (JNICALL *SetStaticBooleanField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jboolean value); + void (JNICALL *SetStaticByteField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jbyte value); + void (JNICALL *SetStaticCharField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jchar value); + void (JNICALL *SetStaticShortField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jshort value); + void (JNICALL *SetStaticIntField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jint value); + void (JNICALL *SetStaticLongField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jlong value); + void (JNICALL *SetStaticFloatField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jfloat value); + void (JNICALL *SetStaticDoubleField) + (JNIEnv *env, jclass clazz, jfieldID fieldID, jdouble value); + + jstring (JNICALL *NewString) + (JNIEnv *env, const jchar *unicode, jsize len); + jsize (JNICALL *GetStringLength) + (JNIEnv *env, jstring str); + const jchar *(JNICALL *GetStringChars) + (JNIEnv *env, jstring str, jboolean *isCopy); + void (JNICALL *ReleaseStringChars) + (JNIEnv *env, jstring str, const jchar *chars); + + jstring (JNICALL *NewStringUTF) + (JNIEnv *env, const char *utf); + jsize (JNICALL *GetStringUTFLength) + (JNIEnv *env, jstring str); + const char* (JNICALL *GetStringUTFChars) + (JNIEnv *env, jstring str, jboolean *isCopy); + void (JNICALL *ReleaseStringUTFChars) + (JNIEnv *env, jstring str, const char* chars); + + + jsize (JNICALL *GetArrayLength) + (JNIEnv *env, jarray array); + + jobjectArray (JNICALL *NewObjectArray) + (JNIEnv *env, jsize len, jclass clazz, jobject init); + jobject (JNICALL *GetObjectArrayElement) + (JNIEnv *env, jobjectArray array, jsize index); + void (JNICALL *SetObjectArrayElement) + (JNIEnv *env, jobjectArray array, jsize index, jobject val); + + jbooleanArray (JNICALL *NewBooleanArray) + (JNIEnv *env, jsize len); + jbyteArray (JNICALL *NewByteArray) + (JNIEnv *env, jsize len); + jcharArray (JNICALL *NewCharArray) + (JNIEnv *env, jsize len); + jshortArray (JNICALL *NewShortArray) + (JNIEnv *env, jsize len); + jintArray (JNICALL *NewIntArray) + (JNIEnv *env, jsize len); + jlongArray (JNICALL *NewLongArray) + (JNIEnv *env, jsize len); + jfloatArray (JNICALL *NewFloatArray) + (JNIEnv *env, jsize len); + jdoubleArray (JNICALL *NewDoubleArray) + (JNIEnv *env, jsize len); + + jboolean * (JNICALL *GetBooleanArrayElements) + (JNIEnv *env, jbooleanArray array, jboolean *isCopy); + jbyte * (JNICALL *GetByteArrayElements) + (JNIEnv *env, jbyteArray array, jboolean *isCopy); + jchar * (JNICALL *GetCharArrayElements) + (JNIEnv *env, jcharArray array, jboolean *isCopy); + jshort * (JNICALL *GetShortArrayElements) + (JNIEnv *env, jshortArray array, jboolean *isCopy); + jint * (JNICALL *GetIntArrayElements) + (JNIEnv *env, jintArray array, jboolean *isCopy); + jlong * (JNICALL *GetLongArrayElements) + (JNIEnv *env, jlongArray array, jboolean *isCopy); + jfloat * (JNICALL *GetFloatArrayElements) + (JNIEnv *env, jfloatArray array, jboolean *isCopy); + jdouble * (JNICALL *GetDoubleArrayElements) + (JNIEnv *env, jdoubleArray array, jboolean *isCopy); + + void (JNICALL *ReleaseBooleanArrayElements) + (JNIEnv *env, jbooleanArray array, jboolean *elems, jint mode); + void (JNICALL *ReleaseByteArrayElements) + (JNIEnv *env, jbyteArray array, jbyte *elems, jint mode); + void (JNICALL *ReleaseCharArrayElements) + (JNIEnv *env, jcharArray array, jchar *elems, jint mode); + void (JNICALL *ReleaseShortArrayElements) + (JNIEnv *env, jshortArray array, jshort *elems, jint mode); + void (JNICALL *ReleaseIntArrayElements) + (JNIEnv *env, jintArray array, jint *elems, jint mode); + void (JNICALL *ReleaseLongArrayElements) + (JNIEnv *env, jlongArray array, jlong *elems, jint mode); + void (JNICALL *ReleaseFloatArrayElements) + (JNIEnv *env, jfloatArray array, jfloat *elems, jint mode); + void (JNICALL *ReleaseDoubleArrayElements) + (JNIEnv *env, jdoubleArray array, jdouble *elems, jint mode); + + void (JNICALL *GetBooleanArrayRegion) + (JNIEnv *env, jbooleanArray array, jsize start, jsize l, jboolean *buf); + void (JNICALL *GetByteArrayRegion) + (JNIEnv *env, jbyteArray array, jsize start, jsize len, jbyte *buf); + void (JNICALL *GetCharArrayRegion) + (JNIEnv *env, jcharArray array, jsize start, jsize len, jchar *buf); + void (JNICALL *GetShortArrayRegion) + (JNIEnv *env, jshortArray array, jsize start, jsize len, jshort *buf); + void (JNICALL *GetIntArrayRegion) + (JNIEnv *env, jintArray array, jsize start, jsize len, jint *buf); + void (JNICALL *GetLongArrayRegion) + (JNIEnv *env, jlongArray array, jsize start, jsize len, jlong *buf); + void (JNICALL *GetFloatArrayRegion) + (JNIEnv *env, jfloatArray array, jsize start, jsize len, jfloat *buf); + void (JNICALL *GetDoubleArrayRegion) + (JNIEnv *env, jdoubleArray array, jsize start, jsize len, jdouble *buf); + + void (JNICALL *SetBooleanArrayRegion) + (JNIEnv *env, jbooleanArray array, jsize start, jsize l, const jboolean *buf); + void (JNICALL *SetByteArrayRegion) + (JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte *buf); + void (JNICALL *SetCharArrayRegion) + (JNIEnv *env, jcharArray array, jsize start, jsize len, const jchar *buf); + void (JNICALL *SetShortArrayRegion) + (JNIEnv *env, jshortArray array, jsize start, jsize len, const jshort *buf); + void (JNICALL *SetIntArrayRegion) + (JNIEnv *env, jintArray array, jsize start, jsize len, const jint *buf); + void (JNICALL *SetLongArrayRegion) + (JNIEnv *env, jlongArray array, jsize start, jsize len, const jlong *buf); + void (JNICALL *SetFloatArrayRegion) + (JNIEnv *env, jfloatArray array, jsize start, jsize len, const jfloat *buf); + void (JNICALL *SetDoubleArrayRegion) + (JNIEnv *env, jdoubleArray array, jsize start, jsize len, const jdouble *buf); + + jint (JNICALL *RegisterNatives) + (JNIEnv *env, jclass clazz, const JNINativeMethod *methods, + jint nMethods); + jint (JNICALL *UnregisterNatives) + (JNIEnv *env, jclass clazz); + + jint (JNICALL *MonitorEnter) + (JNIEnv *env, jobject obj); + jint (JNICALL *MonitorExit) + (JNIEnv *env, jobject obj); + + jint (JNICALL *GetJavaVM) + (JNIEnv *env, JavaVM **vm); + + void (JNICALL *GetStringRegion) + (JNIEnv *env, jstring str, jsize start, jsize len, jchar *buf); + void (JNICALL *GetStringUTFRegion) + (JNIEnv *env, jstring str, jsize start, jsize len, char *buf); + + void * (JNICALL *GetPrimitiveArrayCritical) + (JNIEnv *env, jarray array, jboolean *isCopy); + void (JNICALL *ReleasePrimitiveArrayCritical) + (JNIEnv *env, jarray array, void *carray, jint mode); + + const jchar * (JNICALL *GetStringCritical) + (JNIEnv *env, jstring string, jboolean *isCopy); + void (JNICALL *ReleaseStringCritical) + (JNIEnv *env, jstring string, const jchar *cstring); + + jweak (JNICALL *NewWeakGlobalRef) + (JNIEnv *env, jobject obj); + void (JNICALL *DeleteWeakGlobalRef) + (JNIEnv *env, jweak ref); + + jboolean (JNICALL *ExceptionCheck) + (JNIEnv *env); + + jobject (JNICALL *NewDirectByteBuffer) + (JNIEnv* env, void* address, jlong capacity); + void* (JNICALL *GetDirectBufferAddress) + (JNIEnv* env, jobject buf); + jlong (JNICALL *GetDirectBufferCapacity) + (JNIEnv* env, jobject buf); + + /* New JNI 1.6 Features */ + + jobjectRefType (JNICALL *GetObjectRefType) + (JNIEnv* env, jobject obj); + + /* Module Features */ + + jobject (JNICALL *GetModule) + (JNIEnv* env, jclass clazz); + + /* Virtual threads */ + + jboolean (JNICALL *IsVirtualThread) + (JNIEnv* env, jobject obj); +}; + +/* + * We use inlined functions for C++ so that programmers can write: + * + * env->FindClass("java/lang/String") + * + * in C++ rather than: + * + * (*env)->FindClass(env, "java/lang/String") + * + * in C. + */ + +struct JNIEnv_ { + const struct JNINativeInterface_ *functions; +#ifdef __cplusplus + + jint GetVersion() { + return functions->GetVersion(this); + } + jclass DefineClass(const char *name, jobject loader, const jbyte *buf, + jsize len) { + return functions->DefineClass(this, name, loader, buf, len); + } + jclass FindClass(const char *name) { + return functions->FindClass(this, name); + } + jmethodID FromReflectedMethod(jobject method) { + return functions->FromReflectedMethod(this,method); + } + jfieldID FromReflectedField(jobject field) { + return functions->FromReflectedField(this,field); + } + + jobject ToReflectedMethod(jclass cls, jmethodID methodID, jboolean isStatic) { + return functions->ToReflectedMethod(this, cls, methodID, isStatic); + } + + jclass GetSuperclass(jclass sub) { + return functions->GetSuperclass(this, sub); + } + jboolean IsAssignableFrom(jclass sub, jclass sup) { + return functions->IsAssignableFrom(this, sub, sup); + } + + jobject ToReflectedField(jclass cls, jfieldID fieldID, jboolean isStatic) { + return functions->ToReflectedField(this,cls,fieldID,isStatic); + } + + jint Throw(jthrowable obj) { + return functions->Throw(this, obj); + } + jint ThrowNew(jclass clazz, const char *msg) { + return functions->ThrowNew(this, clazz, msg); + } + jthrowable ExceptionOccurred() { + return functions->ExceptionOccurred(this); + } + void ExceptionDescribe() { + functions->ExceptionDescribe(this); + } + void ExceptionClear() { + functions->ExceptionClear(this); + } + void FatalError(const char *msg) { + functions->FatalError(this, msg); + } + + jint PushLocalFrame(jint capacity) { + return functions->PushLocalFrame(this,capacity); + } + jobject PopLocalFrame(jobject result) { + return functions->PopLocalFrame(this,result); + } + + jobject NewGlobalRef(jobject lobj) { + return functions->NewGlobalRef(this,lobj); + } + void DeleteGlobalRef(jobject gref) { + functions->DeleteGlobalRef(this,gref); + } + void DeleteLocalRef(jobject obj) { + functions->DeleteLocalRef(this, obj); + } + + jboolean IsSameObject(jobject obj1, jobject obj2) { + return functions->IsSameObject(this,obj1,obj2); + } + + jobject NewLocalRef(jobject ref) { + return functions->NewLocalRef(this,ref); + } + jint EnsureLocalCapacity(jint capacity) { + return functions->EnsureLocalCapacity(this,capacity); + } + + jobject AllocObject(jclass clazz) { + return functions->AllocObject(this,clazz); + } + jobject NewObject(jclass clazz, jmethodID methodID, ...) { + va_list args; + jobject result; + va_start(args, methodID); + result = functions->NewObjectV(this,clazz,methodID,args); + va_end(args); + return result; + } + jobject NewObjectV(jclass clazz, jmethodID methodID, + va_list args) { + return functions->NewObjectV(this,clazz,methodID,args); + } + jobject NewObjectA(jclass clazz, jmethodID methodID, + const jvalue *args) { + return functions->NewObjectA(this,clazz,methodID,args); + } + + jclass GetObjectClass(jobject obj) { + return functions->GetObjectClass(this,obj); + } + jboolean IsInstanceOf(jobject obj, jclass clazz) { + return functions->IsInstanceOf(this,obj,clazz); + } + + jmethodID GetMethodID(jclass clazz, const char *name, + const char *sig) { + return functions->GetMethodID(this,clazz,name,sig); + } + + jobject CallObjectMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jobject result; + va_start(args,methodID); + result = functions->CallObjectMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jobject CallObjectMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallObjectMethodV(this,obj,methodID,args); + } + jobject CallObjectMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallObjectMethodA(this,obj,methodID,args); + } + + jboolean CallBooleanMethod(jobject obj, + jmethodID methodID, ...) { + va_list args; + jboolean result; + va_start(args,methodID); + result = functions->CallBooleanMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jboolean CallBooleanMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallBooleanMethodV(this,obj,methodID,args); + } + jboolean CallBooleanMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallBooleanMethodA(this,obj,methodID, args); + } + + jbyte CallByteMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jbyte result; + va_start(args,methodID); + result = functions->CallByteMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jbyte CallByteMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallByteMethodV(this,obj,methodID,args); + } + jbyte CallByteMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallByteMethodA(this,obj,methodID,args); + } + + jchar CallCharMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jchar result; + va_start(args,methodID); + result = functions->CallCharMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jchar CallCharMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallCharMethodV(this,obj,methodID,args); + } + jchar CallCharMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallCharMethodA(this,obj,methodID,args); + } + + jshort CallShortMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jshort result; + va_start(args,methodID); + result = functions->CallShortMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jshort CallShortMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallShortMethodV(this,obj,methodID,args); + } + jshort CallShortMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallShortMethodA(this,obj,methodID,args); + } + + jint CallIntMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jint result; + va_start(args,methodID); + result = functions->CallIntMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jint CallIntMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallIntMethodV(this,obj,methodID,args); + } + jint CallIntMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallIntMethodA(this,obj,methodID,args); + } + + jlong CallLongMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jlong result; + va_start(args,methodID); + result = functions->CallLongMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jlong CallLongMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallLongMethodV(this,obj,methodID,args); + } + jlong CallLongMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallLongMethodA(this,obj,methodID,args); + } + + jfloat CallFloatMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jfloat result; + va_start(args,methodID); + result = functions->CallFloatMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jfloat CallFloatMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallFloatMethodV(this,obj,methodID,args); + } + jfloat CallFloatMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallFloatMethodA(this,obj,methodID,args); + } + + jdouble CallDoubleMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + jdouble result; + va_start(args,methodID); + result = functions->CallDoubleMethodV(this,obj,methodID,args); + va_end(args); + return result; + } + jdouble CallDoubleMethodV(jobject obj, jmethodID methodID, + va_list args) { + return functions->CallDoubleMethodV(this,obj,methodID,args); + } + jdouble CallDoubleMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + return functions->CallDoubleMethodA(this,obj,methodID,args); + } + + void CallVoidMethod(jobject obj, jmethodID methodID, ...) { + va_list args; + va_start(args,methodID); + functions->CallVoidMethodV(this,obj,methodID,args); + va_end(args); + } + void CallVoidMethodV(jobject obj, jmethodID methodID, + va_list args) { + functions->CallVoidMethodV(this,obj,methodID,args); + } + void CallVoidMethodA(jobject obj, jmethodID methodID, + const jvalue * args) { + functions->CallVoidMethodA(this,obj,methodID,args); + } + + jobject CallNonvirtualObjectMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jobject result; + va_start(args,methodID); + result = functions->CallNonvirtualObjectMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jobject CallNonvirtualObjectMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualObjectMethodV(this,obj,clazz, + methodID,args); + } + jobject CallNonvirtualObjectMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualObjectMethodA(this,obj,clazz, + methodID,args); + } + + jboolean CallNonvirtualBooleanMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jboolean result; + va_start(args,methodID); + result = functions->CallNonvirtualBooleanMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jboolean CallNonvirtualBooleanMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualBooleanMethodV(this,obj,clazz, + methodID,args); + } + jboolean CallNonvirtualBooleanMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualBooleanMethodA(this,obj,clazz, + methodID, args); + } + + jbyte CallNonvirtualByteMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jbyte result; + va_start(args,methodID); + result = functions->CallNonvirtualByteMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jbyte CallNonvirtualByteMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualByteMethodV(this,obj,clazz, + methodID,args); + } + jbyte CallNonvirtualByteMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualByteMethodA(this,obj,clazz, + methodID,args); + } + + jchar CallNonvirtualCharMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jchar result; + va_start(args,methodID); + result = functions->CallNonvirtualCharMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jchar CallNonvirtualCharMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualCharMethodV(this,obj,clazz, + methodID,args); + } + jchar CallNonvirtualCharMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualCharMethodA(this,obj,clazz, + methodID,args); + } + + jshort CallNonvirtualShortMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jshort result; + va_start(args,methodID); + result = functions->CallNonvirtualShortMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jshort CallNonvirtualShortMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualShortMethodV(this,obj,clazz, + methodID,args); + } + jshort CallNonvirtualShortMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualShortMethodA(this,obj,clazz, + methodID,args); + } + + jint CallNonvirtualIntMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jint result; + va_start(args,methodID); + result = functions->CallNonvirtualIntMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jint CallNonvirtualIntMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualIntMethodV(this,obj,clazz, + methodID,args); + } + jint CallNonvirtualIntMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualIntMethodA(this,obj,clazz, + methodID,args); + } + + jlong CallNonvirtualLongMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jlong result; + va_start(args,methodID); + result = functions->CallNonvirtualLongMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jlong CallNonvirtualLongMethodV(jobject obj, jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallNonvirtualLongMethodV(this,obj,clazz, + methodID,args); + } + jlong CallNonvirtualLongMethodA(jobject obj, jclass clazz, + jmethodID methodID, const jvalue * args) { + return functions->CallNonvirtualLongMethodA(this,obj,clazz, + methodID,args); + } + + jfloat CallNonvirtualFloatMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jfloat result; + va_start(args,methodID); + result = functions->CallNonvirtualFloatMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jfloat CallNonvirtualFloatMethodV(jobject obj, jclass clazz, + jmethodID methodID, + va_list args) { + return functions->CallNonvirtualFloatMethodV(this,obj,clazz, + methodID,args); + } + jfloat CallNonvirtualFloatMethodA(jobject obj, jclass clazz, + jmethodID methodID, + const jvalue * args) { + return functions->CallNonvirtualFloatMethodA(this,obj,clazz, + methodID,args); + } + + jdouble CallNonvirtualDoubleMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + jdouble result; + va_start(args,methodID); + result = functions->CallNonvirtualDoubleMethodV(this,obj,clazz, + methodID,args); + va_end(args); + return result; + } + jdouble CallNonvirtualDoubleMethodV(jobject obj, jclass clazz, + jmethodID methodID, + va_list args) { + return functions->CallNonvirtualDoubleMethodV(this,obj,clazz, + methodID,args); + } + jdouble CallNonvirtualDoubleMethodA(jobject obj, jclass clazz, + jmethodID methodID, + const jvalue * args) { + return functions->CallNonvirtualDoubleMethodA(this,obj,clazz, + methodID,args); + } + + void CallNonvirtualVoidMethod(jobject obj, jclass clazz, + jmethodID methodID, ...) { + va_list args; + va_start(args,methodID); + functions->CallNonvirtualVoidMethodV(this,obj,clazz,methodID,args); + va_end(args); + } + void CallNonvirtualVoidMethodV(jobject obj, jclass clazz, + jmethodID methodID, + va_list args) { + functions->CallNonvirtualVoidMethodV(this,obj,clazz,methodID,args); + } + void CallNonvirtualVoidMethodA(jobject obj, jclass clazz, + jmethodID methodID, + const jvalue * args) { + functions->CallNonvirtualVoidMethodA(this,obj,clazz,methodID,args); + } + + jfieldID GetFieldID(jclass clazz, const char *name, + const char *sig) { + return functions->GetFieldID(this,clazz,name,sig); + } + + jobject GetObjectField(jobject obj, jfieldID fieldID) { + return functions->GetObjectField(this,obj,fieldID); + } + jboolean GetBooleanField(jobject obj, jfieldID fieldID) { + return functions->GetBooleanField(this,obj,fieldID); + } + jbyte GetByteField(jobject obj, jfieldID fieldID) { + return functions->GetByteField(this,obj,fieldID); + } + jchar GetCharField(jobject obj, jfieldID fieldID) { + return functions->GetCharField(this,obj,fieldID); + } + jshort GetShortField(jobject obj, jfieldID fieldID) { + return functions->GetShortField(this,obj,fieldID); + } + jint GetIntField(jobject obj, jfieldID fieldID) { + return functions->GetIntField(this,obj,fieldID); + } + jlong GetLongField(jobject obj, jfieldID fieldID) { + return functions->GetLongField(this,obj,fieldID); + } + jfloat GetFloatField(jobject obj, jfieldID fieldID) { + return functions->GetFloatField(this,obj,fieldID); + } + jdouble GetDoubleField(jobject obj, jfieldID fieldID) { + return functions->GetDoubleField(this,obj,fieldID); + } + + void SetObjectField(jobject obj, jfieldID fieldID, jobject val) { + functions->SetObjectField(this,obj,fieldID,val); + } + void SetBooleanField(jobject obj, jfieldID fieldID, + jboolean val) { + functions->SetBooleanField(this,obj,fieldID,val); + } + void SetByteField(jobject obj, jfieldID fieldID, + jbyte val) { + functions->SetByteField(this,obj,fieldID,val); + } + void SetCharField(jobject obj, jfieldID fieldID, + jchar val) { + functions->SetCharField(this,obj,fieldID,val); + } + void SetShortField(jobject obj, jfieldID fieldID, + jshort val) { + functions->SetShortField(this,obj,fieldID,val); + } + void SetIntField(jobject obj, jfieldID fieldID, + jint val) { + functions->SetIntField(this,obj,fieldID,val); + } + void SetLongField(jobject obj, jfieldID fieldID, + jlong val) { + functions->SetLongField(this,obj,fieldID,val); + } + void SetFloatField(jobject obj, jfieldID fieldID, + jfloat val) { + functions->SetFloatField(this,obj,fieldID,val); + } + void SetDoubleField(jobject obj, jfieldID fieldID, + jdouble val) { + functions->SetDoubleField(this,obj,fieldID,val); + } + + jmethodID GetStaticMethodID(jclass clazz, const char *name, + const char *sig) { + return functions->GetStaticMethodID(this,clazz,name,sig); + } + + jobject CallStaticObjectMethod(jclass clazz, jmethodID methodID, + ...) { + va_list args; + jobject result; + va_start(args,methodID); + result = functions->CallStaticObjectMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jobject CallStaticObjectMethodV(jclass clazz, jmethodID methodID, + va_list args) { + return functions->CallStaticObjectMethodV(this,clazz,methodID,args); + } + jobject CallStaticObjectMethodA(jclass clazz, jmethodID methodID, + const jvalue *args) { + return functions->CallStaticObjectMethodA(this,clazz,methodID,args); + } + + jboolean CallStaticBooleanMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jboolean result; + va_start(args,methodID); + result = functions->CallStaticBooleanMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jboolean CallStaticBooleanMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticBooleanMethodV(this,clazz,methodID,args); + } + jboolean CallStaticBooleanMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticBooleanMethodA(this,clazz,methodID,args); + } + + jbyte CallStaticByteMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jbyte result; + va_start(args,methodID); + result = functions->CallStaticByteMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jbyte CallStaticByteMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticByteMethodV(this,clazz,methodID,args); + } + jbyte CallStaticByteMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticByteMethodA(this,clazz,methodID,args); + } + + jchar CallStaticCharMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jchar result; + va_start(args,methodID); + result = functions->CallStaticCharMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jchar CallStaticCharMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticCharMethodV(this,clazz,methodID,args); + } + jchar CallStaticCharMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticCharMethodA(this,clazz,methodID,args); + } + + jshort CallStaticShortMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jshort result; + va_start(args,methodID); + result = functions->CallStaticShortMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jshort CallStaticShortMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticShortMethodV(this,clazz,methodID,args); + } + jshort CallStaticShortMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticShortMethodA(this,clazz,methodID,args); + } + + jint CallStaticIntMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jint result; + va_start(args,methodID); + result = functions->CallStaticIntMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jint CallStaticIntMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticIntMethodV(this,clazz,methodID,args); + } + jint CallStaticIntMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticIntMethodA(this,clazz,methodID,args); + } + + jlong CallStaticLongMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jlong result; + va_start(args,methodID); + result = functions->CallStaticLongMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jlong CallStaticLongMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticLongMethodV(this,clazz,methodID,args); + } + jlong CallStaticLongMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticLongMethodA(this,clazz,methodID,args); + } + + jfloat CallStaticFloatMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jfloat result; + va_start(args,methodID); + result = functions->CallStaticFloatMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jfloat CallStaticFloatMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticFloatMethodV(this,clazz,methodID,args); + } + jfloat CallStaticFloatMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticFloatMethodA(this,clazz,methodID,args); + } + + jdouble CallStaticDoubleMethod(jclass clazz, + jmethodID methodID, ...) { + va_list args; + jdouble result; + va_start(args,methodID); + result = functions->CallStaticDoubleMethodV(this,clazz,methodID,args); + va_end(args); + return result; + } + jdouble CallStaticDoubleMethodV(jclass clazz, + jmethodID methodID, va_list args) { + return functions->CallStaticDoubleMethodV(this,clazz,methodID,args); + } + jdouble CallStaticDoubleMethodA(jclass clazz, + jmethodID methodID, const jvalue *args) { + return functions->CallStaticDoubleMethodA(this,clazz,methodID,args); + } + + void CallStaticVoidMethod(jclass cls, jmethodID methodID, ...) { + va_list args; + va_start(args,methodID); + functions->CallStaticVoidMethodV(this,cls,methodID,args); + va_end(args); + } + void CallStaticVoidMethodV(jclass cls, jmethodID methodID, + va_list args) { + functions->CallStaticVoidMethodV(this,cls,methodID,args); + } + void CallStaticVoidMethodA(jclass cls, jmethodID methodID, + const jvalue * args) { + functions->CallStaticVoidMethodA(this,cls,methodID,args); + } + + jfieldID GetStaticFieldID(jclass clazz, const char *name, + const char *sig) { + return functions->GetStaticFieldID(this,clazz,name,sig); + } + jobject GetStaticObjectField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticObjectField(this,clazz,fieldID); + } + jboolean GetStaticBooleanField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticBooleanField(this,clazz,fieldID); + } + jbyte GetStaticByteField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticByteField(this,clazz,fieldID); + } + jchar GetStaticCharField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticCharField(this,clazz,fieldID); + } + jshort GetStaticShortField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticShortField(this,clazz,fieldID); + } + jint GetStaticIntField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticIntField(this,clazz,fieldID); + } + jlong GetStaticLongField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticLongField(this,clazz,fieldID); + } + jfloat GetStaticFloatField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticFloatField(this,clazz,fieldID); + } + jdouble GetStaticDoubleField(jclass clazz, jfieldID fieldID) { + return functions->GetStaticDoubleField(this,clazz,fieldID); + } + + void SetStaticObjectField(jclass clazz, jfieldID fieldID, + jobject value) { + functions->SetStaticObjectField(this,clazz,fieldID,value); + } + void SetStaticBooleanField(jclass clazz, jfieldID fieldID, + jboolean value) { + functions->SetStaticBooleanField(this,clazz,fieldID,value); + } + void SetStaticByteField(jclass clazz, jfieldID fieldID, + jbyte value) { + functions->SetStaticByteField(this,clazz,fieldID,value); + } + void SetStaticCharField(jclass clazz, jfieldID fieldID, + jchar value) { + functions->SetStaticCharField(this,clazz,fieldID,value); + } + void SetStaticShortField(jclass clazz, jfieldID fieldID, + jshort value) { + functions->SetStaticShortField(this,clazz,fieldID,value); + } + void SetStaticIntField(jclass clazz, jfieldID fieldID, + jint value) { + functions->SetStaticIntField(this,clazz,fieldID,value); + } + void SetStaticLongField(jclass clazz, jfieldID fieldID, + jlong value) { + functions->SetStaticLongField(this,clazz,fieldID,value); + } + void SetStaticFloatField(jclass clazz, jfieldID fieldID, + jfloat value) { + functions->SetStaticFloatField(this,clazz,fieldID,value); + } + void SetStaticDoubleField(jclass clazz, jfieldID fieldID, + jdouble value) { + functions->SetStaticDoubleField(this,clazz,fieldID,value); + } + + jstring NewString(const jchar *unicode, jsize len) { + return functions->NewString(this,unicode,len); + } + jsize GetStringLength(jstring str) { + return functions->GetStringLength(this,str); + } + const jchar *GetStringChars(jstring str, jboolean *isCopy) { + return functions->GetStringChars(this,str,isCopy); + } + void ReleaseStringChars(jstring str, const jchar *chars) { + functions->ReleaseStringChars(this,str,chars); + } + + jstring NewStringUTF(const char *utf) { + return functions->NewStringUTF(this,utf); + } + jsize GetStringUTFLength(jstring str) { + return functions->GetStringUTFLength(this,str); + } + const char* GetStringUTFChars(jstring str, jboolean *isCopy) { + return functions->GetStringUTFChars(this,str,isCopy); + } + void ReleaseStringUTFChars(jstring str, const char* chars) { + functions->ReleaseStringUTFChars(this,str,chars); + } + + jsize GetArrayLength(jarray array) { + return functions->GetArrayLength(this,array); + } + + jobjectArray NewObjectArray(jsize len, jclass clazz, + jobject init) { + return functions->NewObjectArray(this,len,clazz,init); + } + jobject GetObjectArrayElement(jobjectArray array, jsize index) { + return functions->GetObjectArrayElement(this,array,index); + } + void SetObjectArrayElement(jobjectArray array, jsize index, + jobject val) { + functions->SetObjectArrayElement(this,array,index,val); + } + + jbooleanArray NewBooleanArray(jsize len) { + return functions->NewBooleanArray(this,len); + } + jbyteArray NewByteArray(jsize len) { + return functions->NewByteArray(this,len); + } + jcharArray NewCharArray(jsize len) { + return functions->NewCharArray(this,len); + } + jshortArray NewShortArray(jsize len) { + return functions->NewShortArray(this,len); + } + jintArray NewIntArray(jsize len) { + return functions->NewIntArray(this,len); + } + jlongArray NewLongArray(jsize len) { + return functions->NewLongArray(this,len); + } + jfloatArray NewFloatArray(jsize len) { + return functions->NewFloatArray(this,len); + } + jdoubleArray NewDoubleArray(jsize len) { + return functions->NewDoubleArray(this,len); + } + + jboolean * GetBooleanArrayElements(jbooleanArray array, jboolean *isCopy) { + return functions->GetBooleanArrayElements(this,array,isCopy); + } + jbyte * GetByteArrayElements(jbyteArray array, jboolean *isCopy) { + return functions->GetByteArrayElements(this,array,isCopy); + } + jchar * GetCharArrayElements(jcharArray array, jboolean *isCopy) { + return functions->GetCharArrayElements(this,array,isCopy); + } + jshort * GetShortArrayElements(jshortArray array, jboolean *isCopy) { + return functions->GetShortArrayElements(this,array,isCopy); + } + jint * GetIntArrayElements(jintArray array, jboolean *isCopy) { + return functions->GetIntArrayElements(this,array,isCopy); + } + jlong * GetLongArrayElements(jlongArray array, jboolean *isCopy) { + return functions->GetLongArrayElements(this,array,isCopy); + } + jfloat * GetFloatArrayElements(jfloatArray array, jboolean *isCopy) { + return functions->GetFloatArrayElements(this,array,isCopy); + } + jdouble * GetDoubleArrayElements(jdoubleArray array, jboolean *isCopy) { + return functions->GetDoubleArrayElements(this,array,isCopy); + } + + void ReleaseBooleanArrayElements(jbooleanArray array, + jboolean *elems, + jint mode) { + functions->ReleaseBooleanArrayElements(this,array,elems,mode); + } + void ReleaseByteArrayElements(jbyteArray array, + jbyte *elems, + jint mode) { + functions->ReleaseByteArrayElements(this,array,elems,mode); + } + void ReleaseCharArrayElements(jcharArray array, + jchar *elems, + jint mode) { + functions->ReleaseCharArrayElements(this,array,elems,mode); + } + void ReleaseShortArrayElements(jshortArray array, + jshort *elems, + jint mode) { + functions->ReleaseShortArrayElements(this,array,elems,mode); + } + void ReleaseIntArrayElements(jintArray array, + jint *elems, + jint mode) { + functions->ReleaseIntArrayElements(this,array,elems,mode); + } + void ReleaseLongArrayElements(jlongArray array, + jlong *elems, + jint mode) { + functions->ReleaseLongArrayElements(this,array,elems,mode); + } + void ReleaseFloatArrayElements(jfloatArray array, + jfloat *elems, + jint mode) { + functions->ReleaseFloatArrayElements(this,array,elems,mode); + } + void ReleaseDoubleArrayElements(jdoubleArray array, + jdouble *elems, + jint mode) { + functions->ReleaseDoubleArrayElements(this,array,elems,mode); + } + + void GetBooleanArrayRegion(jbooleanArray array, + jsize start, jsize len, jboolean *buf) { + functions->GetBooleanArrayRegion(this,array,start,len,buf); + } + void GetByteArrayRegion(jbyteArray array, + jsize start, jsize len, jbyte *buf) { + functions->GetByteArrayRegion(this,array,start,len,buf); + } + void GetCharArrayRegion(jcharArray array, + jsize start, jsize len, jchar *buf) { + functions->GetCharArrayRegion(this,array,start,len,buf); + } + void GetShortArrayRegion(jshortArray array, + jsize start, jsize len, jshort *buf) { + functions->GetShortArrayRegion(this,array,start,len,buf); + } + void GetIntArrayRegion(jintArray array, + jsize start, jsize len, jint *buf) { + functions->GetIntArrayRegion(this,array,start,len,buf); + } + void GetLongArrayRegion(jlongArray array, + jsize start, jsize len, jlong *buf) { + functions->GetLongArrayRegion(this,array,start,len,buf); + } + void GetFloatArrayRegion(jfloatArray array, + jsize start, jsize len, jfloat *buf) { + functions->GetFloatArrayRegion(this,array,start,len,buf); + } + void GetDoubleArrayRegion(jdoubleArray array, + jsize start, jsize len, jdouble *buf) { + functions->GetDoubleArrayRegion(this,array,start,len,buf); + } + + void SetBooleanArrayRegion(jbooleanArray array, jsize start, jsize len, + const jboolean *buf) { + functions->SetBooleanArrayRegion(this,array,start,len,buf); + } + void SetByteArrayRegion(jbyteArray array, jsize start, jsize len, + const jbyte *buf) { + functions->SetByteArrayRegion(this,array,start,len,buf); + } + void SetCharArrayRegion(jcharArray array, jsize start, jsize len, + const jchar *buf) { + functions->SetCharArrayRegion(this,array,start,len,buf); + } + void SetShortArrayRegion(jshortArray array, jsize start, jsize len, + const jshort *buf) { + functions->SetShortArrayRegion(this,array,start,len,buf); + } + void SetIntArrayRegion(jintArray array, jsize start, jsize len, + const jint *buf) { + functions->SetIntArrayRegion(this,array,start,len,buf); + } + void SetLongArrayRegion(jlongArray array, jsize start, jsize len, + const jlong *buf) { + functions->SetLongArrayRegion(this,array,start,len,buf); + } + void SetFloatArrayRegion(jfloatArray array, jsize start, jsize len, + const jfloat *buf) { + functions->SetFloatArrayRegion(this,array,start,len,buf); + } + void SetDoubleArrayRegion(jdoubleArray array, jsize start, jsize len, + const jdouble *buf) { + functions->SetDoubleArrayRegion(this,array,start,len,buf); + } + + jint RegisterNatives(jclass clazz, const JNINativeMethod *methods, + jint nMethods) { + return functions->RegisterNatives(this,clazz,methods,nMethods); + } + jint UnregisterNatives(jclass clazz) { + return functions->UnregisterNatives(this,clazz); + } + + jint MonitorEnter(jobject obj) { + return functions->MonitorEnter(this,obj); + } + jint MonitorExit(jobject obj) { + return functions->MonitorExit(this,obj); + } + + jint GetJavaVM(JavaVM **vm) { + return functions->GetJavaVM(this,vm); + } + + void GetStringRegion(jstring str, jsize start, jsize len, jchar *buf) { + functions->GetStringRegion(this,str,start,len,buf); + } + void GetStringUTFRegion(jstring str, jsize start, jsize len, char *buf) { + functions->GetStringUTFRegion(this,str,start,len,buf); + } + + void * GetPrimitiveArrayCritical(jarray array, jboolean *isCopy) { + return functions->GetPrimitiveArrayCritical(this,array,isCopy); + } + void ReleasePrimitiveArrayCritical(jarray array, void *carray, jint mode) { + functions->ReleasePrimitiveArrayCritical(this,array,carray,mode); + } + + const jchar * GetStringCritical(jstring string, jboolean *isCopy) { + return functions->GetStringCritical(this,string,isCopy); + } + void ReleaseStringCritical(jstring string, const jchar *cstring) { + functions->ReleaseStringCritical(this,string,cstring); + } + + jweak NewWeakGlobalRef(jobject obj) { + return functions->NewWeakGlobalRef(this,obj); + } + void DeleteWeakGlobalRef(jweak ref) { + functions->DeleteWeakGlobalRef(this,ref); + } + + jboolean ExceptionCheck() { + return functions->ExceptionCheck(this); + } + + jobject NewDirectByteBuffer(void* address, jlong capacity) { + return functions->NewDirectByteBuffer(this, address, capacity); + } + void* GetDirectBufferAddress(jobject buf) { + return functions->GetDirectBufferAddress(this, buf); + } + jlong GetDirectBufferCapacity(jobject buf) { + return functions->GetDirectBufferCapacity(this, buf); + } + jobjectRefType GetObjectRefType(jobject obj) { + return functions->GetObjectRefType(this, obj); + } + + /* Module Features */ + + jobject GetModule(jclass clazz) { + return functions->GetModule(this, clazz); + } + + /* Virtual threads */ + + jboolean IsVirtualThread(jobject obj) { + return functions->IsVirtualThread(this, obj); + } + +#endif /* __cplusplus */ +}; + +/* + * optionString may be any option accepted by the JVM, or one of the + * following: + * + * -D= Set a system property. + * -verbose[:class|gc|jni] Enable verbose output, comma-separated. E.g. + * "-verbose:class" or "-verbose:gc,class" + * Standard names include: gc, class, and jni. + * All nonstandard (VM-specific) names must begin + * with "X". + * vfprintf extraInfo is a pointer to the vfprintf hook. + * exit extraInfo is a pointer to the exit hook. + * abort extraInfo is a pointer to the abort hook. + */ +typedef struct JavaVMOption { + char *optionString; + void *extraInfo; +} JavaVMOption; + +typedef struct JavaVMInitArgs { + jint version; + + jint nOptions; + JavaVMOption *options; + jboolean ignoreUnrecognized; +} JavaVMInitArgs; + +typedef struct JavaVMAttachArgs { + jint version; + + char *name; + jobject group; +} JavaVMAttachArgs; + +/* These will be VM-specific. */ + +#define JDK1_2 +#define JDK1_4 + +/* End VM-specific. */ + +struct JNIInvokeInterface_ { + void *reserved0; + void *reserved1; + void *reserved2; + + jint (JNICALL *DestroyJavaVM)(JavaVM *vm); + + jint (JNICALL *AttachCurrentThread)(JavaVM *vm, void **penv, void *args); + + jint (JNICALL *DetachCurrentThread)(JavaVM *vm); + + jint (JNICALL *GetEnv)(JavaVM *vm, void **penv, jint version); + + jint (JNICALL *AttachCurrentThreadAsDaemon)(JavaVM *vm, void **penv, void *args); +}; + +struct JavaVM_ { + const struct JNIInvokeInterface_ *functions; +#ifdef __cplusplus + + jint DestroyJavaVM() { + return functions->DestroyJavaVM(this); + } + jint AttachCurrentThread(void **penv, void *args) { + return functions->AttachCurrentThread(this, penv, args); + } + jint DetachCurrentThread() { + return functions->DetachCurrentThread(this); + } + + jint GetEnv(void **penv, jint version) { + return functions->GetEnv(this, penv, version); + } + jint AttachCurrentThreadAsDaemon(void **penv, void *args) { + return functions->AttachCurrentThreadAsDaemon(this, penv, args); + } +#endif +}; + +#ifdef _JNI_IMPLEMENTATION_ +#define _JNI_IMPORT_OR_EXPORT_ JNIEXPORT +#else +#define _JNI_IMPORT_OR_EXPORT_ JNIIMPORT +#endif +_JNI_IMPORT_OR_EXPORT_ jint JNICALL +JNI_GetDefaultJavaVMInitArgs(void *args); + +_JNI_IMPORT_OR_EXPORT_ jint JNICALL +JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args); + +_JNI_IMPORT_OR_EXPORT_ jint JNICALL +JNI_GetCreatedJavaVMs(JavaVM **, jsize, jsize *); + +/* Defined by native libraries. */ +JNIEXPORT jint JNICALL +JNI_OnLoad(JavaVM *vm, void *reserved); + +JNIEXPORT void JNICALL +JNI_OnUnload(JavaVM *vm, void *reserved); + +#define JNI_VERSION_1_1 0x00010001 +#define JNI_VERSION_1_2 0x00010002 +#define JNI_VERSION_1_4 0x00010004 +#define JNI_VERSION_1_6 0x00010006 +#define JNI_VERSION_1_8 0x00010008 +#define JNI_VERSION_9 0x00090000 +#define JNI_VERSION_10 0x000a0000 +#define JNI_VERSION_19 0x00130000 +#define JNI_VERSION_20 0x00140000 +#define JNI_VERSION_21 0x00150000 + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* !_JAVASOFT_JNI_H_ */ diff --git a/.github/include/windows/jni_md.h b/.github/include/windows/jni_md.h new file mode 100644 index 000000000..6c8d6b9e7 --- /dev/null +++ b/.github/include/windows/jni_md.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 1996, 1998, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef _JAVASOFT_JNI_MD_H_ +#define _JAVASOFT_JNI_MD_H_ + +#define JNIEXPORT __declspec(dllexport) +#define JNIIMPORT __declspec(dllimport) +#define JNICALL __stdcall + +// 'long' is always 32 bit on windows so this matches what jdk expects +typedef long jint; +typedef __int64 jlong; +typedef signed char jbyte; + +#endif /* !_JAVASOFT_JNI_MD_H_ */ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..a15f809d1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,105 @@ +--- +name: Continuous Integration +on: + - pull_request + - workflow_dispatch +env: + MODEL_URL: https://huggingface.co/TheBloke/CodeLlama-7B-GGUF/resolve/main/codellama-7b.Q2_K.gguf + MODEL_NAME: codellama-7b.Q2_K.gguf + RERANKING_MODEL_URL: https://huggingface.co/gpustack/jina-reranker-v1-tiny-en-GGUF/resolve/main/jina-reranker-v1-tiny-en-Q4_0.gguf + RERANKING_MODEL_NAME: jina-reranker-v1-tiny-en-Q4_0.gguf +jobs: + + build-and-test-linux: + name: ubuntu-latest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: "11" + - name: Build libraries + run: | + mvn compile + .github/build.sh -DLLAMA_VERBOSE=ON + - name: Download text generation model + run: curl -L ${MODEL_URL} --create-dirs -o models/${MODEL_NAME} + - name: Download reranking model + run: curl -L ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} + - name: List files in models directory + run: ls -l models/ + - name: Run tests + run: mvn test + - if: failure() + uses: actions/upload-artifact@v4 + with: + name: error-log-linux + path: ${{ github.workspace }}/hs_err_pid*.log + if-no-files-found: warn + + build-and-test-macos: + name: ${{ matrix.target.runner }} + runs-on: ${{ matrix.target.runner }} + strategy: + fail-fast: false + matrix: + target: + - runner: macos-13 + cmake: -DLLAMA_METAL=OFF -DLLAMA_VERBOSE=ON + - runner: macos-14 + cmake: -DLLAMA_METAL_EMBED_LIBRARY=ON -DLLAMA_VERBOSE=ON + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: "11" + - name: Build libraries + run: | + mvn compile + .github/build.sh ${{ matrix.target.cmake }} + - name: Download text generaton model model + run: curl -L ${MODEL_URL} --create-dirs -o models/${MODEL_NAME} + - name: Download reranking model + run: curl -L ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} + - name: List files in models directory + run: ls -l models/ + - name: Run tests + run: mvn test + - if: failure() + uses: actions/upload-artifact@v4 + with: + name: error-log-macos + path: ${{ github.workspace }}/hs_err_pid*.log + if-no-files-found: warn + + build-and-test-windows: + name: windows-2019 + runs-on: windows-2019 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '11' + - name: Build libraries + run: | + mvn compile + .github\build.bat -DLLAMA_VERBOSE=ON + - name: Download model + run: curl -L $env:MODEL_URL --create-dirs -o models/$env:MODEL_NAME + - name: Download reranking model + run: curl -L $env:RERANKING_MODEL_URL --create-dirs -o models/$env:RERANKING_MODEL_NAME + - name: List files in models directory + run: ls -l models/ + - name: Run tests + run: mvn test + - if: failure() + uses: actions/upload-artifact@v4 + with: + name: windows-output + path: | + ${{ github.workspace }}\hs_err_pid*.log + ${{ github.workspace }}/src/main/resources/de/kherud/llama/**/* + if-no-files-found: warn diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 82a742d0c..640320285 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,13 +1,217 @@ -name: Maven Central Publish +name: Release to Maven Central on: workflow_dispatch: + inputs: + build_only: + description: 'Whether to only build the project and skip releasing it (yes/NO)' + required: false + default: 'no' release: - types: [created] + types: [ created ] +env: + MODEL_URL: "https://huggingface.co/TheBloke/CodeLlama-7B-GGUF/resolve/main/codellama-7b.Q2_K.gguf" + MODEL_NAME: "codellama-7b.Q2_K.gguf" + RERANKING_MODEL_URL: "https://huggingface.co/gpustack/jina-reranker-v1-tiny-en-GGUF/resolve/main/jina-reranker-v1-tiny-en-Q4_0.gguf" + RERANKING_MODEL_NAME: "jina-reranker-v1-tiny-en-Q4_0.gguf" jobs: + +# todo: doesn't work with the newest llama.cpp version +# build-linux-cuda: +# name: Build Linux x86-64 CUDA12 +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - name: Build libraries +# shell: bash +# run: | +# .github/dockcross/dockcross-manylinux_2_28-x64 .github/build_cuda_linux.sh "-DOS_NAME=Linux -DOS_ARCH=x86_64" +# - name: Upload artifacts +# uses: actions/upload-artifact@v4 +# with: +# name: linux-libraries-cuda +# path: ${{ github.workspace }}/src/main/resources_linux_cuda/de/kherud/llama/ + + build-linux-docker: + name: Build ${{ matrix.target.os }}-${{ matrix.target.arch }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: + - { + os: Linux, + arch: x86_64, + image: dockcross-manylinux2014-x64, + } + - { + os: Linux, + arch: aarch64, + image: dockcross-linux-arm64-lts, + } + - { + os: Linux-Android, + arch: aarch64, + image: dockcross-android-arm64, + } + steps: + - uses: actions/checkout@v4 + - name: Build libraries + shell: bash + run: | + .github/dockcross/${{ matrix.target.image }} .github/build.sh "-DOS_NAME=${{ matrix.target.os }} -DOS_ARCH=${{ matrix.target.arch }}" + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.target.os }}-${{ matrix.target.arch }}-libraries + path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ + + + build-macos-native: + name: Build ${{ matrix.target.runner }} + runs-on: ${{ matrix.target.runner }} + strategy: + fail-fast: false + matrix: + target: + - { + runner: macos-13, + cmake: '-DLLAMA_METAL=OFF' + } + - { + runner: macos-14, + cmake: '-DLLAMA_METAL_EMBED_LIBRARY=ON' + } + steps: + - uses: actions/checkout@v4 + - name: Build libraries + shell: bash + run: | + mvn compile + .github/build.sh ${{ matrix.target.cmake }} + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.target.runner }}-libraries + path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ + + + build-win-native: + name: Build ${{ matrix.target.os }}-${{ matrix.target.arch }} + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + target: + - { + os: Windows, + arch: x86_64, + cmake: '-G "Visual Studio 16 2019" -A "x64"' + } + - { + os: Windows, + arch: x86, + cmake: '-G "Visual Studio 16 2019" -A "Win32"' + } +# MSVC aarch64 builds no longer work with llama.cpp (requires clang instead) +# - { +# os: Windows, +# arch: aarch64, +# cmake: '-G "Visual Studio 16 2019" -A "ARM64"' +# } +# - { +# os: Windows, +# arch: arm, +# cmake: '-G "Visual Studio 16 2019" -A "ARM"' +# } + steps: + - uses: actions/checkout@v4 + - name: Build libraries + shell: cmd + run: | + .github\build.bat ${{ matrix.target.cmake }} -DOS_NAME=${{ matrix.target.os }} -DOS_ARCH=${{ matrix.target.arch }} + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.target.os }}-${{ matrix.target.arch }}-libraries + path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ + + + test-linux: + name: Test Linux + needs: build-linux-docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: Linux-x86_64-libraries + path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ + - name: Download text generation model + run: curl -L ${MODEL_URL} --create-dirs -o models/${MODEL_NAME} + - name: Download reranking model + run: curl -L ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} + - uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '11' + - name: Run tests + run: mvn test + +# test-macos: +# name: Test Mac +# needs: build-macos-native +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/download-artifact@v4 +# with: +# name: macos14-libraries +# path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ +# - name: Download model +# run: curl -L ${MODEL_URL} --create-dirs -o models/${MODEL_NAME} +# - uses: actions/setup-java@v4 +# with: +# distribution: 'zulu' +# java-version: '11' +# - name: Run tests +# run: mvn test + + +# test-windows: +# name: Test Windows +# needs: build-win-native +# runs-on: windows-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/download-artifact@v4 +# with: +# name: Windows-x86_64-libraries +# path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ +# - name: Download model +# run: curl -L $env:MODEL_URL --create-dirs -o models/$env:MODEL_NAME +# - uses: actions/setup-java@v4 +# with: +# distribution: 'zulu' +# java-version: '11' +# - name: Run tests +# run: mvn test + + publish: + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.build_only == 'no' }} + needs: [ test-linux,build-macos-native,build-win-native ] #,build-linux-cuda runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: "*-libraries" + merge-multiple: true + path: ${{ github.workspace }}/src/main/resources/de/kherud/llama/ +# - uses: actions/download-artifact@v4 +# with: +# name: linux-libraries-cuda +# path: ${{ github.workspace }}/src/main/resources_linux_cuda/de/kherud/llama/ - name: Set up Maven Central Repository uses: actions/setup-java@v3 with: @@ -19,7 +223,7 @@ jobs: gpg-private-key: ${{ secrets.GPG_SIGNING_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Publish package - run: mvn --batch-mode -P release deploy + run: mvn --batch-mode -P release -Dmaven.test.skip=true deploy env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} diff --git a/.gitignore b/.gitignore index 5bea1a5fb..274f86874 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ .idea target +build +cmake-build-* .DS_Store +.directory +.vscode # Compiled class file *.class @@ -27,10 +31,15 @@ target hs_err_pid* replay_pid* -llama.h -llama.cpp -main.cpp - -*.so -*.dylib -*.metal \ No newline at end of file +models/*.gguf +src/main/cpp/de_kherud_llama_*.h +src/main/resources_cuda_linux/ +src/main/resources/**/*.so +src/main/resources/**/*.dylib +src/main/resources/**/*.dll +src/main/resources/**/*.metal +src/test/resources/**/*.gbnf + +**/*.etag +**/*.lastModified +src/main/cpp/llama.cpp/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..96c62950f --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,121 @@ +cmake_minimum_required(VERSION 3.14) + +project(jllama CXX) + +include(FetchContent) + +set(BUILD_SHARED_LIBS ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(BUILD_SHARED_LIBS OFF) + +option(LLAMA_VERBOSE "llama: verbose output" OFF) + +#################### json #################### + +FetchContent_Declare( + json + GIT_REPOSITORY https://github.com/nlohmann/json + GIT_TAG v3.11.3 +) +FetchContent_MakeAvailable(json) + +#################### llama.cpp #################### + +set(LLAMA_BUILD_COMMON ON) +FetchContent_Declare( + llama.cpp + GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git + GIT_TAG b4916 +) +FetchContent_MakeAvailable(llama.cpp) + +#################### jllama #################### + +# find which OS we build for if not set (make sure to run mvn compile first) +if(NOT DEFINED OS_NAME) + find_package(Java REQUIRED) + find_program(JAVA_EXECUTABLE NAMES java) + execute_process( + COMMAND ${JAVA_EXECUTABLE} -cp ${CMAKE_SOURCE_DIR}/target/classes de.kherud.llama.OSInfo --os + OUTPUT_VARIABLE OS_NAME + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() +if(NOT OS_NAME) + message(FATAL_ERROR "Could not determine OS name") +endif() + +# find which architecture we build for if not set (make sure to run mvn compile first) +if(NOT DEFINED OS_ARCH) + find_package(Java REQUIRED) + find_program(JAVA_EXECUTABLE NAMES java) + execute_process( + COMMAND ${JAVA_EXECUTABLE} -cp ${CMAKE_SOURCE_DIR}/target/classes de.kherud.llama.OSInfo --arch + OUTPUT_VARIABLE OS_ARCH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() +if(NOT OS_ARCH) + message(FATAL_ERROR "Could not determine CPU architecture") +endif() + +if(GGML_CUDA) + set(JLLAMA_DIR ${CMAKE_SOURCE_DIR}/src/main/resources_linux_cuda/de/kherud/llama/${OS_NAME}/${OS_ARCH}) + message(STATUS "GPU (CUDA Linux) build - Installing files to ${JLLAMA_DIR}") +else() + set(JLLAMA_DIR ${CMAKE_SOURCE_DIR}/src/main/resources/de/kherud/llama/${OS_NAME}/${OS_ARCH}) + message(STATUS "CPU build - Installing files to ${JLLAMA_DIR}") +endif() + +# include jni.h and jni_md.h +if(NOT DEFINED JNI_INCLUDE_DIRS) + if(OS_NAME MATCHES "^Linux" OR OS_NAME STREQUAL "Mac" OR OS_NAME STREQUAL "Darwin") + set(JNI_INCLUDE_DIRS .github/include/unix) + elseif(OS_NAME STREQUAL "Windows") + set(JNI_INCLUDE_DIRS .github/include/windows) + # if we don't have provided headers, try to find them via Java + else() + find_package(Java REQUIRED) + find_program(JAVA_EXECUTABLE NAMES java) + + find_path(JNI_INCLUDE_DIRS NAMES jni.h HINTS ENV JAVA_HOME PATH_SUFFIXES include) + + # find "jni_md.h" include directory if not set + file(GLOB_RECURSE JNI_MD_PATHS RELATIVE "${JNI_INCLUDE_DIRS}" "${JNI_INCLUDE_DIRS}/**/jni_md.h") + foreach(PATH IN LISTS JNI_MD_PATHS) + get_filename_component(DIR ${PATH} DIRECTORY) + list(APPEND JNI_INCLUDE_DIRS "${JNI_INCLUDE_DIRS}/${DIR}") + endforeach() + endif() +endif() +if(NOT JNI_INCLUDE_DIRS) + message(FATAL_ERROR "Could not determine JNI include directories") +endif() + +add_library(jllama SHARED src/main/cpp/jllama.cpp src/main/cpp/server.hpp src/main/cpp/utils.hpp) + +set_target_properties(jllama PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_include_directories(jllama PRIVATE src/main/cpp ${JNI_INCLUDE_DIRS}) +target_link_libraries(jllama PRIVATE common llama nlohmann_json) +target_compile_features(jllama PRIVATE cxx_std_11) + +target_compile_definitions(jllama PRIVATE + SERVER_VERBOSE=$ +) + +if(OS_NAME STREQUAL "Windows") + set_target_properties(jllama llama ggml PROPERTIES + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${JLLAMA_DIR} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${JLLAMA_DIR} + RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${JLLAMA_DIR} + ) +else() + set_target_properties(jllama llama ggml PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${JLLAMA_DIR} + ) +endif() + +if (LLAMA_METAL AND NOT LLAMA_METAL_EMBED_LIBRARY) + # copy ggml-common.h and ggml-metal.metal to bin directory + configure_file(${llama.cpp_SOURCE_DIR}/ggml-metal.metal ${JLLAMA_DIR}/ggml-metal.metal COPYONLY) +endif() diff --git a/README.md b/README.md index f16c4f3a8..1bc278b10 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,24 @@ ![Java 11+](https://img.shields.io/badge/Java-11%2B-informational) -![llama.cpp b1170](https://img.shields.io/badge/llama.cpp-%23b1204-informational) +![llama.cpp b4916](https://img.shields.io/badge/llama.cpp-%23b4916-informational) # Java Bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) -The main goal of llama.cpp is to run the LLaMA model using 4-bit integer quantization on a MacBook. -This repository provides Java bindings for the C++ library. +Inference of Meta's LLaMA model (and others) in pure C/C++. **You are welcome to contribute** +1. [Quick Start](#quick-start) + 1.1 [No Setup required](#no-setup-required) + 1.2 [Setup required](#setup-required) +2. [Documentation](#documentation) + 2.1 [Example](#example) + 2.2 [Inference](#inference) + 2.3 [Infilling](#infilling) +3. [Android](#importing-in-android) + +> [!NOTE] +> Now with support for Gemma 3 + ## Quick Start Access this library via Maven: @@ -16,184 +27,232 @@ Access this library via Maven: de.kherud llama - 1.1.1 + 4.1.0 ``` -You can then use this library. This is a short example: +There are multiple [examples](src/test/java/examples). + +### No Setup required + +We support CPU inference for the following platforms out of the box: + +- Linux x86-64, aarch64 +- MacOS x86-64, aarch64 (M-series) +- Windows x86-64, x64 + +If any of these match your platform, you can include the Maven dependency and get started. + +### Setup required + +If none of the above listed platforms matches yours, currently you have to compile the library yourself (also if you +want GPU acceleration). + +This consists of two steps: 1) Compiling the libraries and 2) putting them in the right location. + +##### Library Compilation + +First, have a look at [llama.cpp](https://github.com/ggerganov/llama.cpp/blob/master/docs/build.md) to know which build arguments to use (e.g. for CUDA support). +Any build option of llama.cpp works equivalently for this project. +You then have to run the following commands in the directory of this repository (java-llama.cpp): + +```shell +mvn compile # don't forget this line +cmake -B build # add any other arguments for your backend, e.g. -DGGML_CUDA=ON +cmake --build build --config Release +``` + +> [!TIP] +> Use `-DLLAMA_CURL=ON` to download models via Java code using `ModelParameters#setModelUrl(String)`. + +All compiled libraries will be put in a resources directory matching your platform, which will appear in the cmake output. For example something like: + +```shell +-- Installing files to /java-llama.cpp/src/main/resources/de/kherud/llama/Linux/x86_64 +``` + +#### Library Location + +This project has to load a single shared library `jllama`. + +Note, that the file name varies between operating systems, e.g., `jllama.dll` on Windows, `jllama.so` on Linux, and `jllama.dylib` on macOS. + +The application will search in the following order in the following locations: + +- In **de.kherud.llama.lib.path**: Use this option if you want a custom location for your shared libraries, i.e., set VM option `-Dde.kherud.llama.lib.path=/path/to/directory`. +- In **java.library.path**: These are predefined locations for each OS, e.g., `/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib` on Linux. + You can find out the locations using `System.out.println(System.getProperty("java.library.path"))`. + Use this option if you want to install the shared libraries as system libraries. +- From the **JAR**: If any of the libraries weren't found yet, the application will try to use a prebuilt shared library. + This of course only works for the [supported platforms](#no-setup-required) . + +## Documentation + +### Example + +This is a short example on how to use this library: ```java public class Example { public static void main(String... args) throws IOException { - Parameters params = new Parameters.Builder() - .setNGpuLayers(43) - .setTemperature(0.7f) - .setPenalizeNl(true) - .setMirostat(Parameters.MiroStat.V2) - .setAntiPrompt(new String[]{"\n"}) - .build(); - - String modelPath = "/path/to/gguf-model-q4_0.bin"; + ModelParameters modelParams = new ModelParameters() + .setModel("models/mistral-7b-instruct-v0.2.Q2_K.gguf") + .setGpuLayers(43); + String system = "This is a conversation between User and Llama, a friendly chatbot.\n" + "Llama is helpful, kind, honest, good at writing, and never fails to answer any " + "requests immediately and with precision.\n"; BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); - try (LlamaModel model = new LlamaModel(modelPath, params)) { + try (LlamaModel model = new LlamaModel(modelParams)) { + System.out.print(system); String prompt = system; while (true) { prompt += "\nUser: "; - System.out.print(prompt); + System.out.print("\nUser: "); String input = reader.readLine(); prompt += input; System.out.print("Llama: "); prompt += "\nLlama: "; - for (LlamaModel.Output output : model.generate(prompt)) { + InferenceParameters inferParams = new InferenceParameters(prompt) + .setTemperature(0.7f) + .setPenalizeNl(true) + .setMiroStat(MiroStat.V2) + .setStopStrings("User:"); + for (LlamaOutput output : model.generate(inferParams)) { System.out.print(output); + prompt += output; } - prompt = ""; } } } } ``` -Also have a look at the [examples](src/test/java/examples). - -### Installing the llama.cpp library - -Make sure the `llama.cpp` shared library is appropriately installed for your platform: - -- `libllama.so` (linux) -- `libllama.dylib` (macos) -- `llama.dll` (windows) - -Refer to the official [readme](https://github.com/ggerganov/llama.cpp#build) for details. -The library can be built with the `llama.cpp` project: - -```shell -mkdir build -cd build -cmake .. -DBUILD_SHARED_LIBS=ON # add any other arguments for your backend -cmake --build . --config Release -``` - -Look for the shared library in `build`. - -> [!IMPORTANT] -> If you are running MacOS with Metal, you have to put the file `ggml-metal.metal` from `build/bin` in the same directory as the shared library. - -Depending on your platform, either: - -- Move the file then to the correct directory, e.g., `/usr/local/lib` for most linux distributions. -If you're not sure where to put it, just run the code. Java will throw an error explaining where it looks. -- Set the JVM option `-Djna.library.path="/path/to/library/"` (IDEs like IntelliJ make this easy) - -## Documentation - +Also have a look at the other [examples](src/test/java/examples). ### Inference -There are multiple inference tasks. In general, `LlamaModel` is stateful, i.e., unless `LlamaModel#reset()` is called, -each subsequent call takes the previous requests and responses into context. +There are multiple inference tasks. In general, `LlamaModel` is stateless, i.e., you have to append the output of the +model to your prompt in order to extend the context. If there is repeated content, however, the library will internally +cache this, to improve performance. ```java -try (LlamaModel model = new LlamaModel("/path/to/gguf-model")) { +ModelParameters modelParams = new ModelParameters().setModel("/path/to/model.gguf"); +InferenceParameters inferParams = new InferenceParameters("Tell me a joke."); +try (LlamaModel model = new LlamaModel(modelParams)) { // Stream a response and access more information about each output. - for (LlamaModel.Output output : model.generate("Tell me a joke.")) { + for (LlamaOutput output : model.generate(inferParams)) { System.out.print(output); } // Calculate a whole response before returning it. - String response = model.complete("Tell me another one"); + String response = model.complete(inferParams); // Returns the hidden representation of the context + prompt. - float[] embedding = model.getEmbedding("Embed this"); + float[] embedding = model.embed("Embed this"); } ``` -If the model runs out of context at any point, it truncates the saved context to keep half of the maximal context size. - > [!NOTE] > Since llama.cpp allocates memory that can't be garbage collected by the JVM, `LlamaModel` is implemented as an > AutoClosable. If you use the objects with `try-with` blocks like the examples, the memory will be automatically > freed when the model is no longer needed. This isn't strictly required, but avoids memory leaks if you use different > models throughout the lifecycle of your application. -### Model Information +### Infilling -There is some information you can access of your loaded model. +You can simply set `InferenceParameters#setInputPrefix(String)` and `InferenceParameters#setInputSuffix(String)`. -```java -try (LlamaModel model = new LlamaModel("/path/to/gguf-model")) { - // the maximal amount of tokens this model can process - int contextSize = model.getContextSize(); - // the hidden dimensionality of this model - int embeddingSize = model.getEmbeddingSize(); - // the total amount of tokens known in the vocabulary - int vocabularySize = model.getVocabularySize(); - // the tokenization method of the model, i.e., sentence piece or byte pair encoding - VocabularyType vocabType = model.getVocabularyType(); -} -``` +### Model/Inference Configuration -### Model Configuration - -You can configure most options the library offers. -Note however that most options aren't relevant to this Java binding yet (in particular everything that concerns command line interfacing). +There are two sets of parameters you can configure, `ModelParameters` and `InferenceParameters`. Both provide builder +classes to ease configuration. `ModelParameters` are once needed for loading a model, `InferenceParameters` are needed +for every inference task. All non-specified options have sensible defaults. ```java -Parameters params = new Parameters.Builder() - .setInputPrefix("...") - .setLoraAdapter("/path/to/lora/adapter") - .setLoraBase("/path/to/lora/base") - .build(); -LlamaModel model = new LlamaModel("/path/to/model.bin", params); +ModelParameters modelParams = new ModelParameters() + .setModel("/path/to/model.gguf") + .addLoraAdapter("/path/to/lora/adapter"); +String grammar = """ + root ::= (expr "=" term "\\n")+ + expr ::= term ([-+*/] term)* + term ::= [0-9]"""; +InferenceParameters inferParams = new InferenceParameters("") + .setGrammar(grammar) + .setTemperature(0.8); +try (LlamaModel model = new LlamaModel(modelParams)) { + model.generate(inferParams); +} ``` ### Logging -Both Java and C++ logging can be configured via the static method `LlamaModel.setLogger`: +Per default, logs are written to stdout. +This can be intercepted via the static method `LlamaModel.setLogger(LogFormat, BiConsumer)`. +There is text- and JSON-based logging. The default is JSON. +Note, that text-based logging will include additional output of the GGML backend, while JSON-based logging +only provides request logs (while still writing GGML messages to stdout). +To only change the log format while still writing to stdout, `null` can be passed for the callback. +Logging can be disabled by passing an empty callback. ```java -// The method accepts a BiConsumer. -LlamaModel.setLogger((level, message) -> System.out.println(level.name() + ": " + message)); -// This can also be set to null to disable Java logging. -// However, in this case the C++ side will still output to stdout/stderr. -LlamaModel.setLogger(null); -// To completely silence any output, pass a no-op. -LlamaModel.setLogger((level, message) -> {}); - -// Similarly, a progress callback can be set (only the C++ side will call this). -// I think this is only used to report progress loading the model with a value of 0-1. -// It is thus state specific and can be done via the parameters. -new Parameters.Builder() - .setProgressCallback(progress -> System.out.println("progress: " + progress)) - .build(); +// Re-direct log messages however you like (e.g. to a logging library) +LlamaModel.setLogger(LogFormat.TEXT, (level, message) -> System.out.println(level.name() + ": " + message)); +// Log to stdout, but change the format +LlamaModel.setLogger(LogFormat.TEXT, null); +// Disable logging by passing a no-op +LlamaModel.setLogger(null, (level, message) -> {}); ``` -### Debugging Information +## Importing in Android -There are some methods to debug your shared library: - -```java -// Returns some information like "AVX = 1 | AVX2 = 1 | AVX512 = 0 | ...". -String systemInfo = LlamaLibrary.llama_print_system_info(); -// I think this returns the amount of logical cores llama.cpp can use (not completely sure though). -int maxDevices = llama_max_devices(); -// These two methods return a C bool, which is a byte in Java (0 = false, >0 = true). -boolean mmapSupported = llama_mmap_supported() > 0; -boolean mLockSupported = llama_mlock_supported() > 0; -// Returns a timestamp of the llama.cpp backend -long time = LlamaLibrary.llama_time_us(); +You can use this library in Android project. +1. Add java-llama.cpp as a submodule in your an droid `app` project directory +```shell +git submodule add https://github.com/kherud/java-llama.cpp ``` +2. Declare the library as a source in your build.gradle +```gradle +android { + val jllamaLib = file("java-llama.cpp") + + // Execute "mvn compile" if folder target/ doesn't exist at ./java-llama.cpp/ + if (!file("$jllamaLib/target").exists()) { + exec { + commandLine = listOf("mvn", "compile") + workingDir = file("java-llama.cpp/") + } + } -### Backend + ... + defaultConfig { + ... + externalNativeBuild { + cmake { + // Add an flags if needed + cppFlags += "" + arguments += "" + } + } + } -The `llama.cpp` backend is statically initialized upon accessing `LlamaModel`. If you want to de-allocate and maybe -later re-initialize it for whatever reason, there are two methods: + // Declare c++ sources + externalNativeBuild { + cmake { + path = file("$jllamaLib/CMakeLists.txt") + version = "3.22.1" + } + } + // Declare java sources + sourceSets { + named("main") { + // Add source directory for java-llama.cpp + java.srcDir("$jllamaLib/src/main/java") + } + } +} ``` -// This method takes a bool (byte, 0 = false, >0 = true) to enable NUMA optimizations. -// Per default, they are off. If you want to enable them, first free the backend, then initialize it again with 1. -LLamaLibrary.llama_backend_init((byte) 0); -LLamaLibrary.llama_backend_free(); +3. Exclude `de.kherud.llama` in proguard-rules.pro +```proguard +keep class de.kherud.llama.** { *; } ``` diff --git a/models/README.md b/models/README.md new file mode 100644 index 000000000..2481356b7 --- /dev/null +++ b/models/README.md @@ -0,0 +1,3 @@ +# Local Model Directory +This directory contains models which will be automatically downloaded +for use in java-llama.cpp's unit tests. diff --git a/pom.xml b/pom.xml index 5fd2bf32a..67b366ee5 100644 --- a/pom.xml +++ b/pom.xml @@ -1,14 +1,16 @@ - 4.0.0 de.kherud llama - 1.1.1 + 4.2.0 jar ${project.groupId}:${project.artifactId} - Java Bindings for llama.cpp - A Port of Facebook's LLaMA model in C/C++. + Java Bindings for llama.cpp - A Port of Facebook's LLaMA model + in C/C++. https://github.com/kherud/java-llama.cpp @@ -39,27 +41,18 @@ ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ 5.13.0 - 4.13.1 + 4.13.2 UTF-8 - - net.java.dev.jna - jna - ${jna.version} - - - net.java.dev.jna - jna-platform - ${jna.version} - junit junit @@ -69,7 +62,7 @@ org.jetbrains annotations - 24.0.1 + 24.1.0 compile @@ -79,12 +72,55 @@ org.apache.maven.plugins maven-compiler-plugin - 3.11.0 - - 11 - 11 - + 3.13.0 + + + + gpu + compile + + compile + + + + -h + src/main/cpp + + + ${project.build.outputDirectory}_cuda + + + + + maven-resources-plugin + 3.3.1 + + + + copy-resources + process-classes + + copy-resources + + + + ${project.build.outputDirectory}_cuda + + + + ${basedir}/src/main/resources_linux_cuda/ + + **/*.* + + + + + + + + @@ -144,6 +180,27 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + cuda + package + + jar + + + cuda12-linux-x86-64 + + ${project.build.outputDirectory}_cuda + + + + diff --git a/scripts/build-dylib.sh b/scripts/build-dylib.sh deleted file mode 100755 index 35e97cd73..000000000 --- a/scripts/build-dylib.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -cd .. -git clone https://github.com/ggerganov/llama.cpp.git -cd llama.cpp -cp llama.h .. -mkdir build -cd build -cmake -DLLAMA_METAL=ON -DBUILD_SHARED_LIBS=ON .. -cmake --build . --config Release -cd ../.. -mv -f llama.cpp/build/libllama.dylib src/main/resources/ -mv -f llama.cpp/build/bin/ggml-metal.metal src/main/resources/ -rm -rf llama.cpp -cd scripts \ No newline at end of file diff --git a/scripts/build-so.sh b/scripts/build-so.sh deleted file mode 100755 index 3d31ebe9e..000000000 --- a/scripts/build-so.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -cd .. -git clone https://github.com/ggerganov/llama.cpp.git -cd llama.cpp -cp llama.h .. -mkdir build -cd build -cmake -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=ON .. -cmake --build . --config Release -cd ../.. -mv -f llama.cpp/build/libllama.so src/main/resources/ -rm -rf llama.cpp -cd scripts diff --git a/scripts/jnaerator.sh b/scripts/jnaerator.sh deleted file mode 100755 index 683b04e7a..000000000 --- a/scripts/jnaerator.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -cd .. -jnaerator \ - -beanStructs \ - -direct \ - -emptyStructsAsForwardDecls \ - -forceStringSignatures \ - -library llama \ - -package de.kherud.llama.foreign \ - -limitComments \ - -skipDeprecated \ - -noComments \ - -o src/main/java \ - -preferJavac \ - -mode Directory \ - -runtime JNA \ - -noPrimitiveArrays \ - -forceStringSignatures \ - -f \ - llama.h -# remove erroneous method signatures -find src/main/java/de/kherud/llama/foreign \ - -type f \ - -name "*.java" \ - -exec sed -i 's/protected List getFieldOrder()/@Override\n\tprotected List getFieldOrder()/g' {} + -# remove unknown imports -find src/main/java/de/kherud/llama/foreign \ - -type f \ - -name "*.java" \ - -exec sed -i -E '/import com\.ochafik\.lang\.jnaerator\.runtime\.NativeSize(ByReference)?;/d' {} + -# fix invalid java doc strings (important for javadoc maven plugin) -find src/main/java/de/kherud/llama/foreign \ - -type f \ - -name "*.java" \ - -exec perl -i -pe 's|\* \@see (?!LlamaLibrary\.)|* \@see LlamaLibrary.|g' {} + -cd scripts diff --git a/src/main/cpp/jllama.cpp b/src/main/cpp/jllama.cpp new file mode 100644 index 000000000..11c80ae0b --- /dev/null +++ b/src/main/cpp/jllama.cpp @@ -0,0 +1,853 @@ +#include "jllama.h" + +#include "arg.h" +#include "json-schema-to-grammar.h" +#include "llama.h" +#include "log.h" +#include "nlohmann/json.hpp" +#include "server.hpp" + +#include +#include +#include + +// We store some references to Java classes and their fields/methods here to speed up things for later and to fail +// early on if anything can't be found. This happens when the JVM loads the shared library (see `JNI_OnLoad`). +// The references remain valid throughout the whole life of the shared library, on `JNI_OnUnload` they are released. + +namespace { +JavaVM *g_vm = nullptr; + +// classes +jclass c_llama_model = nullptr; +jclass c_llama_iterator = nullptr; +jclass c_standard_charsets = nullptr; +jclass c_output = nullptr; +jclass c_string = nullptr; +jclass c_hash_map = nullptr; +jclass c_map = nullptr; +jclass c_set = nullptr; +jclass c_entry = nullptr; +jclass c_iterator = nullptr; +jclass c_integer = nullptr; +jclass c_float = nullptr; +jclass c_biconsumer = nullptr; +jclass c_llama_error = nullptr; +jclass c_log_level = nullptr; +jclass c_log_format = nullptr; +jclass c_error_oom = nullptr; + +// constructors +jmethodID cc_output = nullptr; +jmethodID cc_hash_map = nullptr; +jmethodID cc_integer = nullptr; +jmethodID cc_float = nullptr; + +// methods +jmethodID m_get_bytes = nullptr; +jmethodID m_entry_set = nullptr; +jmethodID m_set_iterator = nullptr; +jmethodID m_iterator_has_next = nullptr; +jmethodID m_iterator_next = nullptr; +jmethodID m_entry_key = nullptr; +jmethodID m_entry_value = nullptr; +jmethodID m_map_put = nullptr; +jmethodID m_int_value = nullptr; +jmethodID m_float_value = nullptr; +jmethodID m_biconsumer_accept = nullptr; + +// fields +jfieldID f_model_pointer = nullptr; +jfieldID f_task_id = nullptr; +jfieldID f_utf_8 = nullptr; +jfieldID f_iter_has_next = nullptr; +jfieldID f_log_level_debug = nullptr; +jfieldID f_log_level_info = nullptr; +jfieldID f_log_level_warn = nullptr; +jfieldID f_log_level_error = nullptr; +jfieldID f_log_format_json = nullptr; +jfieldID f_log_format_text = nullptr; + +// objects +jobject o_utf_8 = nullptr; +jobject o_log_level_debug = nullptr; +jobject o_log_level_info = nullptr; +jobject o_log_level_warn = nullptr; +jobject o_log_level_error = nullptr; +jobject o_log_format_json = nullptr; +jobject o_log_format_text = nullptr; +jobject o_log_callback = nullptr; + +/** + * Convert a Java string to a std::string + */ +std::string parse_jstring(JNIEnv *env, jstring java_string) { + auto *const string_bytes = (jbyteArray)env->CallObjectMethod(java_string, m_get_bytes, o_utf_8); + + auto length = (size_t)env->GetArrayLength(string_bytes); + jbyte *byte_elements = env->GetByteArrayElements(string_bytes, nullptr); + + std::string string = std::string((char *)byte_elements, length); + + env->ReleaseByteArrayElements(string_bytes, byte_elements, JNI_ABORT); + env->DeleteLocalRef(string_bytes); + + return string; +} + +char **parse_string_array(JNIEnv *env, const jobjectArray string_array, const jsize length) { + auto *const result = static_cast(malloc(length * sizeof(char *))); + + if (result == nullptr) { + return nullptr; + } + + for (jsize i = 0; i < length; i++) { + auto *const javaString = static_cast(env->GetObjectArrayElement(string_array, i)); + const char *cString = env->GetStringUTFChars(javaString, nullptr); + result[i] = strdup(cString); + env->ReleaseStringUTFChars(javaString, cString); + } + + return result; +} + +void free_string_array(char **array, jsize length) { + if (array != nullptr) { + for (jsize i = 0; i < length; i++) { + free(array[i]); + } + free(array); + } +} + +/** + * Since Java expects utf16 but std::strings are utf8, we can't directly use `env->NewString` or `env-NewString`, + * but we directly send the bytes and do the conversion in Java. Unfortunately, there isn't a nice/standardized way to + * do this conversion in C++ + */ +jbyteArray parse_jbytes(JNIEnv *env, const std::string &string) { + jsize length = string.size(); // NOLINT(*-narrowing-conversions) + jbyteArray bytes = env->NewByteArray(length); + env->SetByteArrayRegion(bytes, 0, length, reinterpret_cast(string.c_str())); + return bytes; +} + +/** + * Map a llama.cpp log level to its Java enumeration option. + */ +jobject log_level_to_jobject(ggml_log_level level) { + switch (level) { + case GGML_LOG_LEVEL_ERROR: + return o_log_level_error; + case GGML_LOG_LEVEL_WARN: + return o_log_level_warn; + default: + case GGML_LOG_LEVEL_INFO: + return o_log_level_info; + case GGML_LOG_LEVEL_DEBUG: + return o_log_level_debug; + } +} + +/** + * Returns the JNIEnv of the current thread. + */ +JNIEnv *get_jni_env() { + JNIEnv *env = nullptr; + if (g_vm == nullptr || g_vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) != JNI_OK) { + throw std::runtime_error("Thread is not attached to the JVM"); + } + return env; +} + +bool log_json; +std::function log_callback; + +/** + * Invoke the log callback if there is any. + */ +void log_callback_trampoline(ggml_log_level level, const char *text, void *user_data) { + if (log_callback != nullptr) { + log_callback(level, text, user_data); + } +} +} // namespace + +/** + * The VM calls JNI_OnLoad when the native library is loaded (for example, through `System.loadLibrary`). + * `JNI_OnLoad` must return the JNI version needed by the native library. + * In order to use any of the new JNI functions, a native library must export a `JNI_OnLoad` function that returns + * `JNI_VERSION_1_2`. If the native library does not export a JNI_OnLoad function, the VM assumes that the library + * only requires JNI version `JNI_VERSION_1_1`. If the VM does not recognize the version number returned by + `JNI_OnLoad`, the VM will unload the library and act as if the library was never loaded. + */ +JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { + g_vm = vm; + JNIEnv *env = nullptr; + + if (JNI_OK != vm->GetEnv((void **)&env, JNI_VERSION_1_1)) { + goto error; + } + + // find classes + c_llama_model = env->FindClass("de/kherud/llama/LlamaModel"); + c_llama_iterator = env->FindClass("de/kherud/llama/LlamaIterator"); + c_standard_charsets = env->FindClass("java/nio/charset/StandardCharsets"); + c_output = env->FindClass("de/kherud/llama/LlamaOutput"); + c_string = env->FindClass("java/lang/String"); + c_hash_map = env->FindClass("java/util/HashMap"); + c_map = env->FindClass("java/util/Map"); + c_set = env->FindClass("java/util/Set"); + c_entry = env->FindClass("java/util/Map$Entry"); + c_iterator = env->FindClass("java/util/Iterator"); + c_integer = env->FindClass("java/lang/Integer"); + c_float = env->FindClass("java/lang/Float"); + c_biconsumer = env->FindClass("java/util/function/BiConsumer"); + c_llama_error = env->FindClass("de/kherud/llama/LlamaException"); + c_log_level = env->FindClass("de/kherud/llama/LogLevel"); + c_log_format = env->FindClass("de/kherud/llama/args/LogFormat"); + c_error_oom = env->FindClass("java/lang/OutOfMemoryError"); + + if (!(c_llama_model && c_llama_iterator && c_standard_charsets && c_output && c_string && c_hash_map && c_map && + c_set && c_entry && c_iterator && c_integer && c_float && c_biconsumer && c_llama_error && c_log_level && + c_log_format && c_error_oom)) { + goto error; + } + + // create references + c_llama_model = (jclass)env->NewGlobalRef(c_llama_model); + c_llama_iterator = (jclass)env->NewGlobalRef(c_llama_iterator); + c_output = (jclass)env->NewGlobalRef(c_output); + c_string = (jclass)env->NewGlobalRef(c_string); + c_hash_map = (jclass)env->NewGlobalRef(c_hash_map); + c_map = (jclass)env->NewGlobalRef(c_map); + c_set = (jclass)env->NewGlobalRef(c_set); + c_entry = (jclass)env->NewGlobalRef(c_entry); + c_iterator = (jclass)env->NewGlobalRef(c_iterator); + c_integer = (jclass)env->NewGlobalRef(c_integer); + c_float = (jclass)env->NewGlobalRef(c_float); + c_biconsumer = (jclass)env->NewGlobalRef(c_biconsumer); + c_llama_error = (jclass)env->NewGlobalRef(c_llama_error); + c_log_level = (jclass)env->NewGlobalRef(c_log_level); + c_log_format = (jclass)env->NewGlobalRef(c_log_format); + c_error_oom = (jclass)env->NewGlobalRef(c_error_oom); + + // find constructors + cc_output = env->GetMethodID(c_output, "", "([BLjava/util/Map;Z)V"); + cc_hash_map = env->GetMethodID(c_hash_map, "", "()V"); + cc_integer = env->GetMethodID(c_integer, "", "(I)V"); + cc_float = env->GetMethodID(c_float, "", "(F)V"); + + if (!(cc_output && cc_hash_map && cc_integer && cc_float)) { + goto error; + } + + // find methods + m_get_bytes = env->GetMethodID(c_string, "getBytes", "(Ljava/lang/String;)[B"); + m_entry_set = env->GetMethodID(c_map, "entrySet", "()Ljava/util/Set;"); + m_set_iterator = env->GetMethodID(c_set, "iterator", "()Ljava/util/Iterator;"); + m_iterator_has_next = env->GetMethodID(c_iterator, "hasNext", "()Z"); + m_iterator_next = env->GetMethodID(c_iterator, "next", "()Ljava/lang/Object;"); + m_entry_key = env->GetMethodID(c_entry, "getKey", "()Ljava/lang/Object;"); + m_entry_value = env->GetMethodID(c_entry, "getValue", "()Ljava/lang/Object;"); + m_map_put = env->GetMethodID(c_map, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + m_int_value = env->GetMethodID(c_integer, "intValue", "()I"); + m_float_value = env->GetMethodID(c_float, "floatValue", "()F"); + m_biconsumer_accept = env->GetMethodID(c_biconsumer, "accept", "(Ljava/lang/Object;Ljava/lang/Object;)V"); + + if (!(m_get_bytes && m_entry_set && m_set_iterator && m_iterator_has_next && m_iterator_next && m_entry_key && + m_entry_value && m_map_put && m_int_value && m_float_value && m_biconsumer_accept)) { + goto error; + } + + // find fields + f_model_pointer = env->GetFieldID(c_llama_model, "ctx", "J"); + f_task_id = env->GetFieldID(c_llama_iterator, "taskId", "I"); + f_utf_8 = env->GetStaticFieldID(c_standard_charsets, "UTF_8", "Ljava/nio/charset/Charset;"); + f_iter_has_next = env->GetFieldID(c_llama_iterator, "hasNext", "Z"); + f_log_level_debug = env->GetStaticFieldID(c_log_level, "DEBUG", "Lde/kherud/llama/LogLevel;"); + f_log_level_info = env->GetStaticFieldID(c_log_level, "INFO", "Lde/kherud/llama/LogLevel;"); + f_log_level_warn = env->GetStaticFieldID(c_log_level, "WARN", "Lde/kherud/llama/LogLevel;"); + f_log_level_error = env->GetStaticFieldID(c_log_level, "ERROR", "Lde/kherud/llama/LogLevel;"); + f_log_format_json = env->GetStaticFieldID(c_log_format, "JSON", "Lde/kherud/llama/args/LogFormat;"); + f_log_format_text = env->GetStaticFieldID(c_log_format, "TEXT", "Lde/kherud/llama/args/LogFormat;"); + + if (!(f_model_pointer && f_task_id && f_utf_8 && f_iter_has_next && f_log_level_debug && f_log_level_info && + f_log_level_warn && f_log_level_error && f_log_format_json && f_log_format_text)) { + goto error; + } + + o_utf_8 = env->NewStringUTF("UTF-8"); + o_log_level_debug = env->GetStaticObjectField(c_log_level, f_log_level_debug); + o_log_level_info = env->GetStaticObjectField(c_log_level, f_log_level_info); + o_log_level_warn = env->GetStaticObjectField(c_log_level, f_log_level_warn); + o_log_level_error = env->GetStaticObjectField(c_log_level, f_log_level_error); + o_log_format_json = env->GetStaticObjectField(c_log_format, f_log_format_json); + o_log_format_text = env->GetStaticObjectField(c_log_format, f_log_format_text); + + if (!(o_utf_8 && o_log_level_debug && o_log_level_info && o_log_level_warn && o_log_level_error && + o_log_format_json && o_log_format_text)) { + goto error; + } + + o_utf_8 = env->NewGlobalRef(o_utf_8); + o_log_level_debug = env->NewGlobalRef(o_log_level_debug); + o_log_level_info = env->NewGlobalRef(o_log_level_info); + o_log_level_warn = env->NewGlobalRef(o_log_level_warn); + o_log_level_error = env->NewGlobalRef(o_log_level_error); + o_log_format_json = env->NewGlobalRef(o_log_format_json); + o_log_format_text = env->NewGlobalRef(o_log_format_text); + + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + goto error; + } + + llama_backend_init(); + + goto success; + +error: + return JNI_ERR; + +success: + return JNI_VERSION_1_6; +} + +/** + * The VM calls `JNI_OnUnload` when the class loader containing the native library is garbage collected. + * This function can be used to perform cleanup operations. Because this function is called in an unknown context + * (such as from a finalizer), the programmer should be conservative on using Java VM services, and refrain from + * arbitrary Java call-backs. + * Note that `JNI_OnLoad` and `JNI_OnUnload` are two functions optionally supplied by JNI libraries, not exported from + * the VM. + */ +JNIEXPORT void JNICALL JNI_OnUnload(JavaVM *vm, void *reserved) { + JNIEnv *env = nullptr; + + if (JNI_OK != vm->GetEnv((void **)&env, JNI_VERSION_1_6)) { + return; + } + + env->DeleteGlobalRef(c_llama_model); + env->DeleteGlobalRef(c_llama_iterator); + env->DeleteGlobalRef(c_output); + env->DeleteGlobalRef(c_string); + env->DeleteGlobalRef(c_hash_map); + env->DeleteGlobalRef(c_map); + env->DeleteGlobalRef(c_set); + env->DeleteGlobalRef(c_entry); + env->DeleteGlobalRef(c_iterator); + env->DeleteGlobalRef(c_integer); + env->DeleteGlobalRef(c_float); + env->DeleteGlobalRef(c_biconsumer); + env->DeleteGlobalRef(c_llama_error); + env->DeleteGlobalRef(c_log_level); + env->DeleteGlobalRef(c_log_level); + env->DeleteGlobalRef(c_error_oom); + + env->DeleteGlobalRef(o_utf_8); + env->DeleteGlobalRef(o_log_level_debug); + env->DeleteGlobalRef(o_log_level_info); + env->DeleteGlobalRef(o_log_level_warn); + env->DeleteGlobalRef(o_log_level_error); + env->DeleteGlobalRef(o_log_format_json); + env->DeleteGlobalRef(o_log_format_text); + + if (o_log_callback != nullptr) { + env->DeleteGlobalRef(o_log_callback); + } + + llama_backend_free(); +} + +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_loadModel(JNIEnv *env, jobject obj, jobjectArray jparams) { + common_params params; + + const jsize argc = env->GetArrayLength(jparams); + char **argv = parse_string_array(env, jparams, argc); + if (argv == nullptr) { + return; + } + + const auto parsed_params = common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER); + free_string_array(argv, argc); + if (!parsed_params) { + return; + } + + SRV_INF("loading model '%s'\n", params.model.c_str()); + + common_init(); + + // struct that contains llama context and inference + auto *ctx_server = new server_context(); + + llama_numa_init(params.numa); + + LOG_INF("system info: n_threads = %d, n_threads_batch = %d, total_threads = %d\n", params.cpuparams.n_threads, + params.cpuparams_batch.n_threads, std::thread::hardware_concurrency()); + LOG_INF("\n"); + LOG_INF("%s\n", common_params_get_system_info(params).c_str()); + LOG_INF("\n"); + + std::atomic state{SERVER_STATE_LOADING_MODEL}; + + // Necessary similarity of prompt for slot selection + ctx_server->slot_prompt_similarity = params.slot_prompt_similarity; + + LOG_INF("%s: loading model\n", __func__); + + // load the model + if (!ctx_server->load_model(params)) { + llama_backend_free(); + env->ThrowNew(c_llama_error, "could not load model from given file path"); + return; + } + + ctx_server->init(); + state.store(SERVER_STATE_READY); + + LOG_INF("%s: model loaded\n", __func__); + + const auto model_meta = ctx_server->model_meta(); + + if (!params.speculative.model.empty() || !params.speculative.hf_repo.empty()) { + SRV_INF("loading draft model '%s'\n", params.speculative.model.c_str()); + auto params_dft = params; + + params_dft.devices = params.speculative.devices; + params_dft.hf_file = params.speculative.hf_file; + params_dft.hf_repo = params.speculative.hf_repo; + params_dft.model = params.speculative.model; + params_dft.model_url = params.speculative.model_url; + params_dft.n_ctx = params.speculative.n_ctx == 0 ? params.n_ctx / params.n_parallel : params.speculative.n_ctx; + params_dft.n_gpu_layers = params.speculative.n_gpu_layers; + params_dft.n_parallel = 1; + + common_init_result llama_init_dft = common_init_from_params(params_dft); + + llama_model *model_dft = llama_init_dft.model.get(); + + if (model_dft == nullptr) { + SRV_ERR("failed to load draft model, '%s'\n", params.speculative.model.c_str()); + } + + if (!common_speculative_are_compatible(ctx_server->ctx, llama_init_dft.context.get())) { + SRV_ERR("the draft model '%s' is not compatible with the target model '%s'\n", + params.speculative.model.c_str(), params.model.c_str()); + } + + const int n_ctx_dft = llama_n_ctx(llama_init_dft.context.get()); + + ctx_server->cparams_dft = common_context_params_to_llama(params_dft); + ctx_server->cparams_dft.n_batch = n_ctx_dft; + + // force F16 KV cache for the draft model for extra performance + ctx_server->cparams_dft.type_k = GGML_TYPE_F16; + ctx_server->cparams_dft.type_v = GGML_TYPE_F16; + + // the context is not needed - we will create one for each slot + llama_init_dft.context.reset(); + } + + // print sample chat example to make it clear which template is used + LOG_INF("%s: chat template, chat_template: %s, example_format: '%s'\n", __func__, + common_chat_templates_source(ctx_server->chat_templates.get()), + common_chat_format_example(ctx_server->chat_templates.get(), ctx_server->params_base.use_jinja).c_str()); + + // print sample chat example to make it clear which template is used + // LOG_INF("%s: chat template, chat_template: %s, example_format: '%s'\n", __func__, + // common_chat_templates_source(ctx_server->chat_templates.get()), + // common_chat_format_example(*ctx_server->chat_templates.template_default, + // ctx_server->params_base.use_jinja) .c_str()); + + ctx_server->queue_tasks.on_new_task( + std::bind(&server_context::process_single_task, ctx_server, std::placeholders::_1)); + ctx_server->queue_tasks.on_update_slots(std::bind(&server_context::update_slots, ctx_server)); + + std::thread t([ctx_server]() { + JNIEnv *env; + jint res = g_vm->GetEnv((void **)&env, JNI_VERSION_1_6); + if (res == JNI_EDETACHED) { + res = g_vm->AttachCurrentThread((void **)&env, nullptr); + if (res != JNI_OK) { + throw std::runtime_error("Failed to attach thread to JVM"); + } + } + ctx_server->queue_tasks.start_loop(); + }); + t.detach(); + + env->SetLongField(obj, f_model_pointer, reinterpret_cast(ctx_server)); +} + +JNIEXPORT jint JNICALL Java_de_kherud_llama_LlamaModel_requestCompletion(JNIEnv *env, jobject obj, jstring jparams) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + std::string c_params = parse_jstring(env, jparams); + json data = json::parse(c_params); + + server_task_type type = SERVER_TASK_TYPE_COMPLETION; + + if (data.contains("input_prefix") || data.contains("input_suffix")) { + type = SERVER_TASK_TYPE_INFILL; + } + + auto completion_id = gen_chatcmplid(); + std::vector tasks; + + try { + const auto &prompt = data.at("prompt"); + + std::vector tokenized_prompts = tokenize_input_prompts(ctx_server->vocab, prompt, true, true); + + tasks.reserve(tokenized_prompts.size()); + for (size_t i = 0; i < tokenized_prompts.size(); i++) { + server_task task = server_task(type); + + task.id = ctx_server->queue_tasks.get_new_id(); + task.index = i; + + task.prompt_tokens = std::move(tokenized_prompts[i]); + task.params = server_task::params_from_json_cmpl(ctx_server->ctx, ctx_server->params_base, data); + task.id_selected_slot = json_value(data, "id_slot", -1); + + // OAI-compat + task.params.oaicompat = OAICOMPAT_TYPE_NONE; + task.params.oaicompat_cmpl_id = completion_id; + // oaicompat_model is already populated by params_from_json_cmpl + + tasks.push_back(task); + } + } catch (const std::exception &e) { + const auto &err = format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST); + env->ThrowNew(c_llama_error, err.dump().c_str()); + return 0; + } + + ctx_server->queue_results.add_waiting_tasks(tasks); + ctx_server->queue_tasks.post(tasks); + + const auto task_ids = server_task::get_list_id(tasks); + + if (task_ids.size() != 1) { + env->ThrowNew(c_llama_error, "multitasking currently not supported"); + return 0; + } + + return *task_ids.begin(); +} + +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_releaseTask(JNIEnv *env, jobject obj, jint id_task) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + ctx_server->queue_results.remove_waiting_task_id(id_task); +} + +JNIEXPORT jobject JNICALL Java_de_kherud_llama_LlamaModel_receiveCompletion(JNIEnv *env, jobject obj, jint id_task) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + server_task_result_ptr result = ctx_server->queue_results.recv(id_task); + + if (result->is_error()) { + std::string response = result->to_json()["message"].get(); + ctx_server->queue_results.remove_waiting_task_id(id_task); + env->ThrowNew(c_llama_error, response.c_str()); + return nullptr; + } + const auto out_res = result->to_json(); + + std::string response = out_res["content"].get(); + if (result->is_stop()) { + ctx_server->queue_results.remove_waiting_task_id(id_task); + } + + jobject o_probabilities = env->NewObject(c_hash_map, cc_hash_map); + if (out_res.contains("completion_probabilities")) { + auto completion_probabilities = out_res["completion_probabilities"]; + for (const auto &entry : completion_probabilities) { + auto probs = entry["probs"]; + for (const auto &tp : probs) { + std::string tok_str = tp["tok_str"]; + jstring jtok_str = env->NewStringUTF(tok_str.c_str()); + float prob = tp["prob"]; + jobject jprob = env->NewObject(c_float, cc_float, prob); + env->CallObjectMethod(o_probabilities, m_map_put, jtok_str, jprob); + env->DeleteLocalRef(jtok_str); + env->DeleteLocalRef(jprob); + } + } + } + jbyteArray jbytes = parse_jbytes(env, response); + return env->NewObject(c_output, cc_output, jbytes, o_probabilities, result->is_stop()); +} + +JNIEXPORT jfloatArray JNICALL Java_de_kherud_llama_LlamaModel_embed(JNIEnv *env, jobject obj, jstring jprompt) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + if (!ctx_server->params_base.embedding) { + env->ThrowNew(c_llama_error, + "model was not loaded with embedding support (see ModelParameters#setEmbedding(boolean))"); + return nullptr; + } + + const std::string prompt = parse_jstring(env, jprompt); + + SRV_INF("Calling embedding '%s'\n", prompt.c_str()); + + const auto tokens = tokenize_mixed(ctx_server->vocab, prompt, true, true); + std::vector tasks; + + server_task task = server_task(SERVER_TASK_TYPE_EMBEDDING); + + task.id = ctx_server->queue_tasks.get_new_id(); + task.index = 0; + task.prompt_tokens = std::move(tokens); + + // OAI-compat + task.params.oaicompat = OAICOMPAT_TYPE_NONE; + + tasks.push_back(task); + + ctx_server->queue_results.add_waiting_tasks(tasks); + ctx_server->queue_tasks.post(tasks); + + std::unordered_set task_ids = server_task::get_list_id(tasks); + const auto id_task = *task_ids.begin(); + json responses = json::array(); + + json error = nullptr; + + server_task_result_ptr result = ctx_server->queue_results.recv(id_task); + + json response_str = result->to_json(); + if (result->is_error()) { + std::string response = result->to_json()["message"].get(); + ctx_server->queue_results.remove_waiting_task_id(id_task); + env->ThrowNew(c_llama_error, response.c_str()); + return nullptr; + } + + if (result->is_stop()) { + ctx_server->queue_results.remove_waiting_task_id(id_task); + } + + const auto out_res = result->to_json(); + + // Extract "embedding" as a vector of vectors (2D array) + std::vector> embedding = out_res["embedding"].get>>(); + + // Get total number of rows in the embedding + jsize embedding_rows = embedding.size(); + + // Get total number of columns in the first row (assuming all rows are of equal length) + jsize embedding_cols = embedding_rows > 0 ? embedding[0].size() : 0; + + SRV_INF("Embedding has %d rows and %d columns\n", embedding_rows, embedding_cols); + + // Ensure embedding is not empty + if (embedding.empty() || embedding[0].empty()) { + env->ThrowNew(c_error_oom, "embedding array is empty"); + return nullptr; + } + + // Extract only the first row + const std::vector &first_row = embedding[0]; // Reference to avoid copying + + // Create a new float array in JNI + jfloatArray j_embedding = env->NewFloatArray(embedding_cols); + if (j_embedding == nullptr) { + env->ThrowNew(c_error_oom, "could not allocate embedding"); + return nullptr; + } + + // Copy the first row into the JNI float array + env->SetFloatArrayRegion(j_embedding, 0, embedding_cols, reinterpret_cast(first_row.data())); + + return j_embedding; +} + +JNIEXPORT jobject JNICALL Java_de_kherud_llama_LlamaModel_rerank(JNIEnv *env, jobject obj, jstring jprompt, + jobjectArray documents) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + if (!ctx_server->params_base.reranking || ctx_server->params_base.embedding) { + env->ThrowNew(c_llama_error, + "This server does not support reranking. Start it with `--reranking` and without `--embedding`"); + return nullptr; + } + + const std::string prompt = parse_jstring(env, jprompt); + + const auto tokenized_query = tokenize_mixed(ctx_server->vocab, prompt, true, true); + + json responses = json::array(); + + std::vector tasks; + const jsize amount_documents = env->GetArrayLength(documents); + auto *document_array = parse_string_array(env, documents, amount_documents); + auto document_vector = std::vector(document_array, document_array + amount_documents); + free_string_array(document_array, amount_documents); + + std::vector tokenized_docs = tokenize_input_prompts(ctx_server->vocab, document_vector, true, true); + + tasks.reserve(tokenized_docs.size()); + for (int i = 0; i < tokenized_docs.size(); i++) { + auto task = server_task(SERVER_TASK_TYPE_RERANK); + task.id = ctx_server->queue_tasks.get_new_id(); + task.index = i; + task.prompt_tokens = format_rerank(ctx_server->vocab, tokenized_query, tokenized_docs[i]); + tasks.push_back(task); + } + ctx_server->queue_results.add_waiting_tasks(tasks); + ctx_server->queue_tasks.post(tasks); + + // get the result + std::unordered_set task_ids = server_task::get_list_id(tasks); + std::vector results(task_ids.size()); + + // Create a new HashMap instance + jobject o_probabilities = env->NewObject(c_hash_map, cc_hash_map); + if (o_probabilities == nullptr) { + env->ThrowNew(c_llama_error, "Failed to create HashMap object."); + return nullptr; + } + + for (int i = 0; i < (int)task_ids.size(); i++) { + server_task_result_ptr result = ctx_server->queue_results.recv(task_ids); + if (result->is_error()) { + auto response = result->to_json()["message"].get(); + for (const int id_task : task_ids) { + ctx_server->queue_results.remove_waiting_task_id(id_task); + } + env->ThrowNew(c_llama_error, response.c_str()); + return nullptr; + } + + const auto out_res = result->to_json(); + + if (result->is_stop()) { + for (const int id_task : task_ids) { + ctx_server->queue_results.remove_waiting_task_id(id_task); + } + } + + int index = out_res["index"].get(); + float score = out_res["score"].get(); + std::string tok_str = document_vector[index]; + jstring jtok_str = env->NewStringUTF(tok_str.c_str()); + + jobject jprob = env->NewObject(c_float, cc_float, score); + env->CallObjectMethod(o_probabilities, m_map_put, jtok_str, jprob); + env->DeleteLocalRef(jtok_str); + env->DeleteLocalRef(jprob); + } + jbyteArray jbytes = parse_jbytes(env, prompt); + return env->NewObject(c_output, cc_output, jbytes, o_probabilities, true); +} + +JNIEXPORT jstring JNICALL Java_de_kherud_llama_LlamaModel_applyTemplate(JNIEnv *env, jobject obj, jstring jparams) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + std::string c_params = parse_jstring(env, jparams); + json data = json::parse(c_params); + + json templateData = + oaicompat_completion_params_parse(data, ctx_server->params_base.use_jinja, + ctx_server->params_base.reasoning_format, ctx_server->chat_templates.get()); + std::string tok_str = templateData.at("prompt"); + jstring jtok_str = env->NewStringUTF(tok_str.c_str()); + + return jtok_str; +} + +JNIEXPORT jintArray JNICALL Java_de_kherud_llama_LlamaModel_encode(JNIEnv *env, jobject obj, jstring jprompt) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + const std::string c_prompt = parse_jstring(env, jprompt); + + llama_tokens tokens = tokenize_mixed(ctx_server->vocab, c_prompt, false, true); + jsize token_size = tokens.size(); // NOLINT(*-narrowing-conversions) + + jintArray java_tokens = env->NewIntArray(token_size); + if (java_tokens == nullptr) { + env->ThrowNew(c_error_oom, "could not allocate token memory"); + return nullptr; + } + + env->SetIntArrayRegion(java_tokens, 0, token_size, reinterpret_cast(tokens.data())); + + return java_tokens; +} + +JNIEXPORT jbyteArray JNICALL Java_de_kherud_llama_LlamaModel_decodeBytes(JNIEnv *env, jobject obj, + jintArray java_tokens) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + + jsize length = env->GetArrayLength(java_tokens); + jint *elements = env->GetIntArrayElements(java_tokens, nullptr); + std::vector tokens(elements, elements + length); + std::string text = tokens_to_str(ctx_server->ctx, tokens.cbegin(), tokens.cend()); + + env->ReleaseIntArrayElements(java_tokens, elements, 0); + + return parse_jbytes(env, text); +} + +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_delete(JNIEnv *env, jobject obj) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + ctx_server->queue_tasks.terminate(); + // delete ctx_server; +} + +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_cancelCompletion(JNIEnv *env, jobject obj, jint id_task) { + jlong server_handle = env->GetLongField(obj, f_model_pointer); + auto *ctx_server = reinterpret_cast(server_handle); // NOLINT(*-no-int-to-ptr) + std::unordered_set id_tasks = {id_task}; + ctx_server->cancel_tasks(id_tasks); + ctx_server->queue_results.remove_waiting_task_id(id_task); +} + +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_setLogger(JNIEnv *env, jclass clazz, jobject log_format, + jobject jcallback) { + if (o_log_callback != nullptr) { + env->DeleteGlobalRef(o_log_callback); + } + + log_json = env->IsSameObject(log_format, o_log_format_json); + + if (jcallback == nullptr) { + log_callback = nullptr; + llama_log_set(nullptr, nullptr); + } else { + o_log_callback = env->NewGlobalRef(jcallback); + log_callback = [](enum ggml_log_level level, const char *text, void *user_data) { + JNIEnv *env = get_jni_env(); + jstring message = env->NewStringUTF(text); + jobject log_level = log_level_to_jobject(level); + env->CallVoidMethod(o_log_callback, m_biconsumer_accept, log_level, message); + env->DeleteLocalRef(message); + }; + if (!log_json) { + llama_log_set(log_callback_trampoline, nullptr); + } + } +} + +JNIEXPORT jbyteArray JNICALL Java_de_kherud_llama_LlamaModel_jsonSchemaToGrammarBytes(JNIEnv *env, jclass clazz, + jstring j_schema) { + const std::string c_schema = parse_jstring(env, j_schema); + nlohmann::ordered_json c_schema_json = nlohmann::ordered_json::parse(c_schema); + const std::string c_grammar = json_schema_to_grammar(c_schema_json); + return parse_jbytes(env, c_grammar); +} diff --git a/src/main/cpp/jllama.h b/src/main/cpp/jllama.h new file mode 100644 index 000000000..dc17fa831 --- /dev/null +++ b/src/main/cpp/jllama.h @@ -0,0 +1,104 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class de_kherud_llama_LlamaModel */ + +#ifndef _Included_de_kherud_llama_LlamaModel +#define _Included_de_kherud_llama_LlamaModel +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: de_kherud_llama_LlamaModel + * Method: embed + * Signature: (Ljava/lang/String;)[F + */ +JNIEXPORT jfloatArray JNICALL Java_de_kherud_llama_LlamaModel_embed(JNIEnv *, jobject, jstring); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: encode + * Signature: (Ljava/lang/String;)[I + */ +JNIEXPORT jintArray JNICALL Java_de_kherud_llama_LlamaModel_encode(JNIEnv *, jobject, jstring); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: setLogger + * Signature: (Lde/kherud/llama/args/LogFormat;Ljava/util/function/BiConsumer;)V + */ +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_setLogger(JNIEnv *, jclass, jobject, jobject); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: requestCompletion + * Signature: (Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_de_kherud_llama_LlamaModel_requestCompletion(JNIEnv *, jobject, jstring); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: receiveCompletion + * Signature: (I)Lde/kherud/llama/LlamaOutput; + */ +JNIEXPORT jobject JNICALL Java_de_kherud_llama_LlamaModel_receiveCompletion(JNIEnv *, jobject, jint); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: cancelCompletion + * Signature: (I)V + */ +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_cancelCompletion(JNIEnv *, jobject, jint); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: decodeBytes + * Signature: ([I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_de_kherud_llama_LlamaModel_decodeBytes(JNIEnv *, jobject, jintArray); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: loadModel + * Signature: ([Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_loadModel(JNIEnv *, jobject, jobjectArray); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: delete + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_delete(JNIEnv *, jobject); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: releaseTask + * Signature: (I)V + */ +JNIEXPORT void JNICALL Java_de_kherud_llama_LlamaModel_releaseTask(JNIEnv *, jobject, jint); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: jsonSchemaToGrammarBytes + * Signature: (Ljava/lang/String;)[B + */ +JNIEXPORT jbyteArray JNICALL Java_de_kherud_llama_LlamaModel_jsonSchemaToGrammarBytes(JNIEnv *, jclass, jstring); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: rerank + * Signature: (Ljava/lang/String;[Ljava/lang/String;)Lde/kherud/llama/LlamaOutput; + */ +JNIEXPORT jobject JNICALL Java_de_kherud_llama_LlamaModel_rerank(JNIEnv *, jobject, jstring, jobjectArray); + +/* + * Class: de_kherud_llama_LlamaModel + * Method: applyTemplate + * Signature: (Ljava/lang/String;)Ljava/lang/String;; + */ +JNIEXPORT jstring JNICALL Java_de_kherud_llama_LlamaModel_applyTemplate(JNIEnv *, jobject, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/main/cpp/server.hpp b/src/main/cpp/server.hpp new file mode 100644 index 000000000..9686f2af1 --- /dev/null +++ b/src/main/cpp/server.hpp @@ -0,0 +1,3271 @@ +#include "utils.hpp" + +#include "json-schema-to-grammar.h" +#include "sampling.h" +#include "speculative.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::ordered_json; + +constexpr int HTTP_POLLING_SECONDS = 1; + +enum stop_type { + STOP_TYPE_NONE, + STOP_TYPE_EOS, + STOP_TYPE_WORD, + STOP_TYPE_LIMIT, +}; + +// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283 +enum slot_state { + SLOT_STATE_IDLE, + SLOT_STATE_STARTED, // TODO: this state is only used for setting up the initial prompt processing; maybe merge it + // with launch_slot_with_task in the future + SLOT_STATE_PROCESSING_PROMPT, + SLOT_STATE_DONE_PROMPT, + SLOT_STATE_GENERATING, +}; + +enum server_state { + SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet + SERVER_STATE_READY, // Server is ready and model is loaded +}; + +enum server_task_type { + SERVER_TASK_TYPE_COMPLETION, + SERVER_TASK_TYPE_EMBEDDING, + SERVER_TASK_TYPE_RERANK, + SERVER_TASK_TYPE_INFILL, + SERVER_TASK_TYPE_CANCEL, + SERVER_TASK_TYPE_NEXT_RESPONSE, + SERVER_TASK_TYPE_METRICS, + SERVER_TASK_TYPE_SLOT_SAVE, + SERVER_TASK_TYPE_SLOT_RESTORE, + SERVER_TASK_TYPE_SLOT_ERASE, + SERVER_TASK_TYPE_SET_LORA, +}; + +enum oaicompat_type { + OAICOMPAT_TYPE_NONE, + OAICOMPAT_TYPE_CHAT, + OAICOMPAT_TYPE_COMPLETION, + OAICOMPAT_TYPE_EMBEDDING, +}; + +// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11 +enum error_type { + ERROR_TYPE_INVALID_REQUEST, + ERROR_TYPE_AUTHENTICATION, + ERROR_TYPE_SERVER, + ERROR_TYPE_NOT_FOUND, + ERROR_TYPE_PERMISSION, + ERROR_TYPE_UNAVAILABLE, // custom error + ERROR_TYPE_NOT_SUPPORTED, // custom error +}; + +struct slot_params { + bool stream = true; + bool cache_prompt = true; // remember the prompt to avoid reprocessing all prompt + bool return_tokens = false; + + int32_t n_keep = 0; // number of tokens to keep from initial prompt + int32_t n_discard = + 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half + int32_t n_predict = -1; // new tokens to predict + int32_t n_indent = 0; // mininum line indentation for the generated text in number of whitespace characters + + int64_t t_max_prompt_ms = -1; // TODO: implement + int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit + + std::vector lora; + + std::vector antiprompt; + std::vector response_fields; + bool timings_per_token = false; + bool post_sampling_probs = false; + bool ignore_eos = false; + + struct common_params_sampling sampling; + struct common_params_speculative speculative; + + // OAI-compat fields + bool verbose = false; + oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE; + std::string oaicompat_model; + std::string oaicompat_cmpl_id; + common_chat_format oaicompat_chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY; + + json to_json() const { + std::vector samplers; + samplers.reserve(sampling.samplers.size()); + for (const auto &sampler : sampling.samplers) { + samplers.emplace_back(common_sampler_type_to_str(sampler)); + } + + json lora = json::array(); + for (size_t i = 0; i < this->lora.size(); ++i) { + lora.push_back({{"id", i}, {"scale", this->lora[i].scale}}); + } + + auto grammar_triggers = json::array(); + for (const auto &trigger : sampling.grammar_triggers) { + grammar_triggers.push_back(trigger.to_json()); + } + + return json{ + {"n_predict", n_predict}, // Server configured n_predict + {"seed", sampling.seed}, + {"temperature", sampling.temp}, + {"dynatemp_range", sampling.dynatemp_range}, + {"dynatemp_exponent", sampling.dynatemp_exponent}, + {"top_k", sampling.top_k}, + {"top_p", sampling.top_p}, + {"min_p", sampling.min_p}, + {"xtc_probability", sampling.xtc_probability}, + {"xtc_threshold", sampling.xtc_threshold}, + {"typical_p", sampling.typ_p}, + {"repeat_last_n", sampling.penalty_last_n}, + {"repeat_penalty", sampling.penalty_repeat}, + {"presence_penalty", sampling.penalty_present}, + {"frequency_penalty", sampling.penalty_freq}, + {"dry_multiplier", sampling.dry_multiplier}, + {"dry_base", sampling.dry_base}, + {"dry_allowed_length", sampling.dry_allowed_length}, + {"dry_penalty_last_n", sampling.dry_penalty_last_n}, + {"dry_sequence_breakers", sampling.dry_sequence_breakers}, + {"mirostat", sampling.mirostat}, + {"mirostat_tau", sampling.mirostat_tau}, + {"mirostat_eta", sampling.mirostat_eta}, + {"stop", antiprompt}, + {"max_tokens", n_predict}, // User configured n_predict + {"n_keep", n_keep}, + {"n_discard", n_discard}, + {"ignore_eos", sampling.ignore_eos}, + {"stream", stream}, + {"logit_bias", format_logit_bias(sampling.logit_bias)}, + {"n_probs", sampling.n_probs}, + {"min_keep", sampling.min_keep}, + {"grammar", sampling.grammar}, + {"grammar_lazy", sampling.grammar_lazy}, + {"grammar_triggers", grammar_triggers}, + {"preserved_tokens", sampling.preserved_tokens}, + {"chat_format", common_chat_format_name(oaicompat_chat_format)}, + {"samplers", samplers}, + {"speculative.n_max", speculative.n_max}, + {"speculative.n_min", speculative.n_min}, + {"speculative.p_min", speculative.p_min}, + {"timings_per_token", timings_per_token}, + {"post_sampling_probs", post_sampling_probs}, + {"lora", lora}, + }; + } +}; + +struct server_task { + int id = -1; // to be filled by server_queue + int index = -1; // used when there are multiple prompts (batch request) + + server_task_type type; + + // used by SERVER_TASK_TYPE_CANCEL + int id_target = -1; + + // used by SERVER_TASK_TYPE_INFERENCE + slot_params params; + llama_tokens prompt_tokens; + int id_selected_slot = -1; + + // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE + struct slot_action { + int slot_id; + std::string filename; + std::string filepath; + }; + slot_action slot_action; + + // used by SERVER_TASK_TYPE_METRICS + bool metrics_reset_bucket = false; + + // used by SERVER_TASK_TYPE_SET_LORA + std::vector set_lora; + + server_task(server_task_type type) : type(type) {} + + static slot_params params_from_json_cmpl(const llama_context *ctx, const common_params ¶ms_base, + const json &data) { + const llama_model *model = llama_get_model(ctx); + const llama_vocab *vocab = llama_model_get_vocab(model); + + slot_params params; + + // Sampling parameter defaults are loaded from the global server context (but individual requests can still + // override them) + slot_params defaults; + defaults.sampling = params_base.sampling; + defaults.speculative = params_base.speculative; + + // enabling this will output extra debug information in the HTTP responses from the server + params.verbose = params_base.verbosity > 9; + params.timings_per_token = json_value(data, "timings_per_token", false); + + params.stream = json_value(data, "stream", false); + params.cache_prompt = json_value(data, "cache_prompt", true); + params.return_tokens = json_value(data, "return_tokens", false); + params.n_predict = json_value(data, "n_predict", json_value(data, "max_tokens", defaults.n_predict)); + params.n_indent = json_value(data, "n_indent", defaults.n_indent); + params.n_keep = json_value(data, "n_keep", defaults.n_keep); + params.n_discard = json_value(data, "n_discard", defaults.n_discard); + // params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: + // implement + params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); + params.response_fields = json_value(data, "response_fields", std::vector()); + + params.sampling.top_k = json_value(data, "top_k", defaults.sampling.top_k); + params.sampling.top_p = json_value(data, "top_p", defaults.sampling.top_p); + params.sampling.min_p = json_value(data, "min_p", defaults.sampling.min_p); + params.sampling.xtc_probability = json_value(data, "xtc_probability", defaults.sampling.xtc_probability); + params.sampling.xtc_threshold = json_value(data, "xtc_threshold", defaults.sampling.xtc_threshold); + params.sampling.typ_p = json_value(data, "typical_p", defaults.sampling.typ_p); + params.sampling.temp = json_value(data, "temperature", defaults.sampling.temp); + params.sampling.dynatemp_range = json_value(data, "dynatemp_range", defaults.sampling.dynatemp_range); + params.sampling.dynatemp_exponent = json_value(data, "dynatemp_exponent", defaults.sampling.dynatemp_exponent); + params.sampling.penalty_last_n = json_value(data, "repeat_last_n", defaults.sampling.penalty_last_n); + params.sampling.penalty_repeat = json_value(data, "repeat_penalty", defaults.sampling.penalty_repeat); + params.sampling.penalty_freq = json_value(data, "frequency_penalty", defaults.sampling.penalty_freq); + params.sampling.penalty_present = json_value(data, "presence_penalty", defaults.sampling.penalty_present); + params.sampling.dry_multiplier = json_value(data, "dry_multiplier", defaults.sampling.dry_multiplier); + params.sampling.dry_base = json_value(data, "dry_base", defaults.sampling.dry_base); + params.sampling.dry_allowed_length = + json_value(data, "dry_allowed_length", defaults.sampling.dry_allowed_length); + params.sampling.dry_penalty_last_n = + json_value(data, "dry_penalty_last_n", defaults.sampling.dry_penalty_last_n); + params.sampling.mirostat = json_value(data, "mirostat", defaults.sampling.mirostat); + params.sampling.mirostat_tau = json_value(data, "mirostat_tau", defaults.sampling.mirostat_tau); + params.sampling.mirostat_eta = json_value(data, "mirostat_eta", defaults.sampling.mirostat_eta); + params.sampling.seed = json_value(data, "seed", defaults.sampling.seed); + params.sampling.n_probs = json_value(data, "n_probs", defaults.sampling.n_probs); + params.sampling.min_keep = json_value(data, "min_keep", defaults.sampling.min_keep); + params.post_sampling_probs = json_value(data, "post_sampling_probs", defaults.post_sampling_probs); + + params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min); + params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max); + params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min); + + params.speculative.n_min = std::min(params.speculative.n_max, params.speculative.n_min); + params.speculative.n_min = std::max(params.speculative.n_min, 0); + params.speculative.n_max = std::max(params.speculative.n_max, 0); + + // Use OpenAI API logprobs only if n_probs wasn't provided + if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs) { + params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs); + } + + if (data.contains("lora")) { + if (data.at("lora").is_array()) { + params.lora = parse_lora_request(params_base.lora_adapters, data.at("lora")); + } else { + throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields"); + } + } else { + params.lora = params_base.lora_adapters; + } + + // TODO: add more sanity checks for the input parameters + + if (params.sampling.penalty_last_n < -1) { + throw std::runtime_error("Error: repeat_last_n must be >= -1"); + } + + if (params.sampling.dry_penalty_last_n < -1) { + throw std::runtime_error("Error: dry_penalty_last_n must be >= -1"); + } + + if (params.sampling.penalty_last_n == -1) { + // note: should be the slot's context and not the full context, but it's ok + params.sampling.penalty_last_n = llama_n_ctx(ctx); + } + + if (params.sampling.dry_penalty_last_n == -1) { + params.sampling.dry_penalty_last_n = llama_n_ctx(ctx); + } + + if (params.sampling.dry_base < 1.0f) { + params.sampling.dry_base = defaults.sampling.dry_base; + } + + // sequence breakers for DRY + { + // Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format + // Ref: + // https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39 + + if (data.contains("dry_sequence_breakers")) { + params.sampling.dry_sequence_breakers = + json_value(data, "dry_sequence_breakers", std::vector()); + if (params.sampling.dry_sequence_breakers.empty()) { + throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings"); + } + } + } + + // process "json_schema" and "grammar" + if (data.contains("json_schema") && !data.contains("grammar")) { + try { + auto schema = json_value(data, "json_schema", json::object()); + SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str()); + params.sampling.grammar = json_schema_to_grammar(schema); + SRV_DBG("Converted grammar: %s\n", params.sampling.grammar.c_str()); + } catch (const std::exception &e) { + throw std::runtime_error(std::string("\"json_schema\": ") + e.what()); + } + } else { + params.sampling.grammar = json_value(data, "grammar", defaults.sampling.grammar); + SRV_DBG("Grammar: %s\n", params.sampling.grammar.c_str()); + params.sampling.grammar_lazy = json_value(data, "grammar_lazy", defaults.sampling.grammar_lazy); + SRV_DBG("Grammar lazy: %s\n", params.sampling.grammar_lazy ? "true" : "false"); + } + + { + auto it = data.find("chat_format"); + if (it != data.end()) { + params.oaicompat_chat_format = static_cast(it->get()); + SRV_INF("Chat format: %s\n", common_chat_format_name(params.oaicompat_chat_format).c_str()); + } else { + params.oaicompat_chat_format = defaults.oaicompat_chat_format; + } + } + + { + const auto preserved_tokens = data.find("preserved_tokens"); + if (preserved_tokens != data.end()) { + for (const auto &t : *preserved_tokens) { + auto ids = common_tokenize(vocab, t.get(), /* add_special= */ false, + /* parse_special= */ true); + if (ids.size() == 1) { + SRV_DBG("Preserved token: %d\n", ids[0]); + params.sampling.preserved_tokens.insert(ids[0]); + } else { + // This may happen when using a tool call style meant for a model with special tokens to + // preserve on a model without said tokens. + SRV_DBG("Not preserved because more than 1 token: %s\n", t.get().c_str()); + } + } + } + const auto grammar_triggers = data.find("grammar_triggers"); + if (grammar_triggers != data.end()) { + for (const auto &t : *grammar_triggers) { + auto ct = common_grammar_trigger::from_json(t); + if (ct.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) { + const auto &word = ct.value; + auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true); + if (ids.size() == 1) { + auto token = ids[0]; + if (std::find(params.sampling.preserved_tokens.begin(), + params.sampling.preserved_tokens.end(), + (llama_token)token) == params.sampling.preserved_tokens.end()) { + throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + + word); + } + SRV_DBG("Grammar trigger token: %d (`%s`)\n", token, word.c_str()); + common_grammar_trigger trigger; + trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN; + trigger.value = (llama_token)token; + params.sampling.grammar_triggers.push_back(trigger); + } else { + SRV_DBG("Grammar trigger word: `%s`\n", word.c_str()); + params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word}); + } + } else { + params.sampling.grammar_triggers.push_back(ct); + } + } + } + if (params.sampling.grammar_lazy && params.sampling.grammar_triggers.empty()) { + throw std::runtime_error("Error: no triggers set for lazy grammar!"); + } + } + + { + params.sampling.logit_bias.clear(); + params.ignore_eos = json_value(data, "ignore_eos", false); + + const auto &logit_bias = data.find("logit_bias"); + if (logit_bias != data.end() && logit_bias->is_array()) { + const int n_vocab = llama_vocab_n_tokens(vocab); + for (const auto &el : *logit_bias) { + // TODO: we may want to throw errors here, in case "el" is incorrect + if (el.is_array() && el.size() == 2) { + float bias; + if (el[1].is_number()) { + bias = el[1].get(); + } else if (el[1].is_boolean() && !el[1].get()) { + bias = -INFINITY; + } else { + continue; + } + + if (el[0].is_number_integer()) { + llama_token tok = el[0].get(); + if (tok >= 0 && tok < n_vocab) { + params.sampling.logit_bias.push_back({tok, bias}); + } + } else if (el[0].is_string()) { + auto toks = common_tokenize(vocab, el[0].get(), false); + for (auto tok : toks) { + params.sampling.logit_bias.push_back({tok, bias}); + } + } + } + } + } + } + + { + params.antiprompt.clear(); + + const auto &stop = data.find("stop"); + if (stop != data.end() && stop->is_array()) { + for (const auto &word : *stop) { + if (!word.empty()) { + params.antiprompt.push_back(word); + } + } + } + } + + { + const auto samplers = data.find("samplers"); + if (samplers != data.end()) { + if (samplers->is_array()) { + params.sampling.samplers = common_sampler_types_from_names(*samplers, false); + } else if (samplers->is_string()) { + params.sampling.samplers = common_sampler_types_from_chars(samplers->get()); + } + } else { + params.sampling.samplers = defaults.sampling.samplers; + } + } + + std::string model_name = params_base.model_alias.empty() ? DEFAULT_OAICOMPAT_MODEL : params_base.model_alias; + params.oaicompat_model = json_value(data, "model", model_name); + + return params; + } + + // utility function + static std::unordered_set get_list_id(const std::vector &tasks) { + std::unordered_set ids(tasks.size()); + for (size_t i = 0; i < tasks.size(); i++) { + ids.insert(tasks[i].id); + } + return ids; + } +}; + +struct result_timings { + int32_t prompt_n = -1; + double prompt_ms; + double prompt_per_token_ms; + double prompt_per_second; + + int32_t predicted_n = -1; + double predicted_ms; + double predicted_per_token_ms; + double predicted_per_second; + + json to_json() const { + return { + {"prompt_n", prompt_n}, + {"prompt_ms", prompt_ms}, + {"prompt_per_token_ms", prompt_per_token_ms}, + {"prompt_per_second", prompt_per_second}, + + {"predicted_n", predicted_n}, + {"predicted_ms", predicted_ms}, + {"predicted_per_token_ms", predicted_per_token_ms}, + {"predicted_per_second", predicted_per_second}, + }; + } +}; + +struct server_task_result { + int id = -1; + int id_slot = -1; + virtual bool is_error() { + // only used by server_task_result_error + return false; + } + virtual bool is_stop() { + // only used by server_task_result_cmpl_* + return false; + } + virtual int get_index() { return -1; } + virtual json to_json() = 0; + virtual ~server_task_result() = default; +}; + +// using shared_ptr for polymorphism of server_task_result +using server_task_result_ptr = std::unique_ptr; + +inline std::string stop_type_to_str(stop_type type) { + switch (type) { + case STOP_TYPE_EOS: + return "eos"; + case STOP_TYPE_WORD: + return "word"; + case STOP_TYPE_LIMIT: + return "limit"; + default: + return "none"; + } +} + +struct completion_token_output { + llama_token tok; + float prob; + std::string text_to_send; + struct prob_info { + llama_token tok; + std::string txt; + float prob; + }; + std::vector probs; + + json to_json(bool post_sampling_probs) const { + json probs_for_token = json::array(); + for (const auto &p : probs) { + std::string txt(p.txt); + txt.resize(validate_utf8(txt)); + probs_for_token.push_back(json{ + {"id", p.tok}, + {"token", txt}, + {"bytes", str_to_bytes(p.txt)}, + {post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob)}, + }); + } + return probs_for_token; + } + + static json probs_vector_to_json(const std::vector &probs, bool post_sampling_probs) { + json out = json::array(); + for (const auto &p : probs) { + std::string txt(p.text_to_send); + txt.resize(validate_utf8(txt)); + out.push_back(json{ + {"id", p.tok}, + {"token", txt}, + {"bytes", str_to_bytes(p.text_to_send)}, + {post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob)}, + {post_sampling_probs ? "top_probs" : "top_logprobs", p.to_json(post_sampling_probs)}, + }); + } + return out; + } + + static float logarithm(float x) { + // nlohmann::json converts -inf to null, so we need to prevent that + return x == 0.0f ? std::numeric_limits::lowest() : std::log(x); + } + + static std::vector str_to_bytes(const std::string &str) { + std::vector bytes; + for (unsigned char c : str) { + bytes.push_back(c); + } + return bytes; + } +}; + +struct server_task_result_cmpl_final : server_task_result { + int index = 0; + + std::string content; + llama_tokens tokens; + + bool stream; + result_timings timings; + std::string prompt; + + bool truncated; + int32_t n_decoded; + int32_t n_prompt_tokens; + int32_t n_tokens_cached; + bool has_new_line; + std::string stopping_word; + stop_type stop = STOP_TYPE_NONE; + + bool post_sampling_probs; + std::vector probs_output; + std::vector response_fields; + + slot_params generation_params; + + // OAI-compat fields + bool verbose = false; + oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE; + std::string oaicompat_model; + std::string oaicompat_cmpl_id; + common_chat_format oaicompat_chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY; + + virtual int get_index() override { return index; } + + virtual bool is_stop() override { + return true; // in stream mode, final responses are considered stop + } + + virtual json to_json() override { + switch (oaicompat) { + case OAICOMPAT_TYPE_NONE: + return to_json_non_oaicompat(); + case OAICOMPAT_TYPE_COMPLETION: + return to_json_oaicompat(); + case OAICOMPAT_TYPE_CHAT: + return stream ? to_json_oaicompat_chat_stream() : to_json_oaicompat_chat(); + default: + GGML_ASSERT(false && "Invalid oaicompat_type"); + } + } + + json to_json_non_oaicompat() { + json res = json{ + {"index", index}, + {"content", stream ? "" : content}, // in stream mode, content is already in last partial chunk + {"tokens", stream ? llama_tokens{} : tokens}, + {"id_slot", id_slot}, + {"stop", true}, + {"model", oaicompat_model}, + {"tokens_predicted", n_decoded}, + {"tokens_evaluated", n_prompt_tokens}, + {"generation_settings", generation_params.to_json()}, + {"prompt", prompt}, + {"has_new_line", has_new_line}, + {"truncated", truncated}, + {"stop_type", stop_type_to_str(stop)}, + {"stopping_word", stopping_word}, + {"tokens_cached", n_tokens_cached}, + {"timings", timings.to_json()}, + }; + if (!stream && !probs_output.empty()) { + res["completion_probabilities"] = + completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs); + } + return response_fields.empty() ? res : json_get_nested_values(response_fields, res); + } + + json to_json_oaicompat() { + std::time_t t = std::time(0); + json logprobs = json(nullptr); // OAI default to null + if (!stream && probs_output.size() > 0) { + logprobs = json{ + {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)}, + }; + } + json finish_reason = "length"; + if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { + finish_reason = "stop"; + } + json res = json{ + {"choices", json::array({json{ + {"text", stream ? "" : content}, // in stream mode, content is already in last partial chunk + {"index", index}, + {"logprobs", logprobs}, + {"finish_reason", finish_reason}, + }})}, + {"created", t}, + {"model", oaicompat_model}, + {"system_fingerprint", build_info}, + {"object", "text_completion"}, + {"usage", json{{"completion_tokens", n_decoded}, + {"prompt_tokens", n_prompt_tokens}, + {"total_tokens", n_decoded + n_prompt_tokens}}}, + {"id", oaicompat_cmpl_id}}; + + // extra fields for debugging purposes + if (verbose) { + res["__verbose"] = to_json_non_oaicompat(); + } + if (timings.prompt_n >= 0) { + res.push_back({"timings", timings.to_json()}); + } + + return res; + } + + json to_json_oaicompat_chat() { + std::string finish_reason = "length"; + common_chat_msg msg; + if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { + SRV_DBG("Parsing chat message: %s\n", content.c_str()); + msg = common_chat_parse(content, oaicompat_chat_format); + finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls"; + } else { + msg.content = content; + } + + json message{ + {"role", "assistant"}, + }; + if (!msg.reasoning_content.empty()) { + message["reasoning_content"] = msg.reasoning_content; + } + if (msg.content.empty() && !msg.tool_calls.empty()) { + message["content"] = json(); + } else { + message["content"] = msg.content; + } + if (!msg.tool_calls.empty()) { + auto tool_calls = json::array(); + for (const auto &tc : msg.tool_calls) { + tool_calls.push_back({ + {"type", "function"}, + {"function", + { + {"name", tc.name}, + {"arguments", tc.arguments}, + }}, + {"id", tc.id}, + }); + } + message["tool_calls"] = tool_calls; + } + + json choice{ + {"finish_reason", finish_reason}, + {"index", 0}, + {"message", message}, + }; + + if (!stream && probs_output.size() > 0) { + choice["logprobs"] = json{ + {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)}, + }; + } + + std::time_t t = std::time(0); + + json res = json{{"choices", json::array({choice})}, + {"created", t}, + {"model", oaicompat_model}, + {"system_fingerprint", build_info}, + {"object", "chat.completion"}, + {"usage", json{{"completion_tokens", n_decoded}, + {"prompt_tokens", n_prompt_tokens}, + {"total_tokens", n_decoded + n_prompt_tokens}}}, + {"id", oaicompat_cmpl_id}}; + + // extra fields for debugging purposes + if (verbose) { + res["__verbose"] = to_json_non_oaicompat(); + } + if (timings.prompt_n >= 0) { + res.push_back({"timings", timings.to_json()}); + } + + return res; + } + + json to_json_oaicompat_chat_stream() { + std::time_t t = std::time(0); + std::string finish_reason = "length"; + if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { + finish_reason = "stop"; + } + + json choice = json{{"finish_reason", finish_reason}, {"index", 0}, {"delta", json::object()}}; + + json ret = json{ + {"choices", json::array({choice})}, + {"created", t}, + {"id", oaicompat_cmpl_id}, + {"model", oaicompat_model}, + {"system_fingerprint", build_info}, + {"object", "chat.completion.chunk"}, + {"usage", + json{ + {"completion_tokens", n_decoded}, + {"prompt_tokens", n_prompt_tokens}, + {"total_tokens", n_decoded + n_prompt_tokens}, + }}, + }; + + if (timings.prompt_n >= 0) { + ret.push_back({"timings", timings.to_json()}); + } + + return ret; + } +}; + +struct server_task_result_cmpl_partial : server_task_result { + int index = 0; + + std::string content; + llama_tokens tokens; + + int32_t n_decoded; + int32_t n_prompt_tokens; + + bool post_sampling_probs; + completion_token_output prob_output; + result_timings timings; + + // OAI-compat fields + bool verbose = false; + oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE; + std::string oaicompat_model; + std::string oaicompat_cmpl_id; + + virtual int get_index() override { return index; } + + virtual bool is_stop() override { + return false; // in stream mode, partial responses are not considered stop + } + + virtual json to_json() override { + switch (oaicompat) { + case OAICOMPAT_TYPE_NONE: + return to_json_non_oaicompat(); + case OAICOMPAT_TYPE_COMPLETION: + return to_json_oaicompat(); + case OAICOMPAT_TYPE_CHAT: + return to_json_oaicompat_chat(); + default: + GGML_ASSERT(false && "Invalid oaicompat_type"); + } + } + + json to_json_non_oaicompat() { + // non-OAI-compat JSON + json res = json{ + {"index", index}, + {"content", content}, + {"tokens", tokens}, + {"stop", false}, + {"id_slot", id_slot}, + {"tokens_predicted", n_decoded}, + {"tokens_evaluated", n_prompt_tokens}, + }; + // populate the timings object when needed (usually for the last response or with timings_per_token enabled) + if (timings.prompt_n > 0) { + res.push_back({"timings", timings.to_json()}); + } + if (!prob_output.probs.empty()) { + res["completion_probabilities"] = + completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs); + } + return res; + } + + json to_json_oaicompat() { + std::time_t t = std::time(0); + json logprobs = json(nullptr); // OAI default to null + if (prob_output.probs.size() > 0) { + logprobs = json{ + {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)}, + }; + } + json res = json{{"choices", json::array({json{ + {"text", content}, + {"index", index}, + {"logprobs", logprobs}, + {"finish_reason", nullptr}, + }})}, + {"created", t}, + {"model", oaicompat_model}, + {"system_fingerprint", build_info}, + {"object", "text_completion"}, + {"id", oaicompat_cmpl_id}}; + + // extra fields for debugging purposes + if (verbose) { + res["__verbose"] = to_json_non_oaicompat(); + } + if (timings.prompt_n >= 0) { + res.push_back({"timings", timings.to_json()}); + } + + return res; + } + + json to_json_oaicompat_chat() { + bool first = n_decoded == 0; + std::time_t t = std::time(0); + json choices; + + if (first) { + if (content.empty()) { + choices = json::array( + {json{{"finish_reason", nullptr}, {"index", 0}, {"delta", json{{"role", "assistant"}}}}}); + } else { + // We have to send this as two updates to conform to openai behavior + json initial_ret = json{{"choices", json::array({json{{"finish_reason", nullptr}, + {"index", 0}, + {"delta", json{{"role", "assistant"}}}}})}, + {"created", t}, + {"id", oaicompat_cmpl_id}, + {"model", oaicompat_model}, + {"object", "chat.completion.chunk"}}; + + json second_ret = + json{{"choices", + json::array( + {json{{"finish_reason", nullptr}, {"index", 0}, {"delta", json{{"content", content}}}}})}, + {"created", t}, + {"id", oaicompat_cmpl_id}, + {"model", oaicompat_model}, + {"object", "chat.completion.chunk"}}; + + return std::vector({initial_ret, second_ret}); + } + } else { + choices = json::array({json{ + {"finish_reason", nullptr}, + {"index", 0}, + {"delta", + json{ + {"content", content}, + }}, + }}); + } + + GGML_ASSERT(choices.size() >= 1); + + if (prob_output.probs.size() > 0) { + choices[0]["logprobs"] = json{ + {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)}, + }; + } + + json ret = json{{"choices", choices}, + {"created", t}, + {"id", oaicompat_cmpl_id}, + {"model", oaicompat_model}, + {"system_fingerprint", build_info}, + {"object", "chat.completion.chunk"}}; + + if (timings.prompt_n >= 0) { + ret.push_back({"timings", timings.to_json()}); + } + + return std::vector({ret}); + } +}; + +struct server_task_result_embd : server_task_result { + int index = 0; + std::vector> embedding; + + int32_t n_tokens; + + // OAI-compat fields + oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE; + + virtual int get_index() override { return index; } + + virtual json to_json() override { + return oaicompat == OAICOMPAT_TYPE_EMBEDDING ? to_json_oaicompat() : to_json_non_oaicompat(); + } + + json to_json_non_oaicompat() { + return json{ + {"index", index}, + {"embedding", embedding}, + }; + } + + json to_json_oaicompat() { + return json{ + {"index", index}, + {"embedding", embedding[0]}, + {"tokens_evaluated", n_tokens}, + }; + } +}; + +struct server_task_result_rerank : server_task_result { + int index = 0; + float score = -1e6; + + int32_t n_tokens; + + virtual int get_index() override { return index; } + + virtual json to_json() override { + return json{ + {"index", index}, + {"score", score}, + {"tokens_evaluated", n_tokens}, + }; + } +}; + +// this function maybe used outside of server_task_result_error +static json format_error_response(const std::string &message, const enum error_type type) { + std::string type_str; + int code = 500; + switch (type) { + case ERROR_TYPE_INVALID_REQUEST: + type_str = "invalid_request_error"; + code = 400; + break; + case ERROR_TYPE_AUTHENTICATION: + type_str = "authentication_error"; + code = 401; + break; + case ERROR_TYPE_NOT_FOUND: + type_str = "not_found_error"; + code = 404; + break; + case ERROR_TYPE_SERVER: + type_str = "server_error"; + code = 500; + break; + case ERROR_TYPE_PERMISSION: + type_str = "permission_error"; + code = 403; + break; + case ERROR_TYPE_NOT_SUPPORTED: + type_str = "not_supported_error"; + code = 501; + break; + case ERROR_TYPE_UNAVAILABLE: + type_str = "unavailable_error"; + code = 503; + break; + } + return json{ + {"code", code}, + {"message", message}, + {"type", type_str}, + }; +} + +struct server_task_result_error : server_task_result { + int index = 0; + error_type err_type = ERROR_TYPE_SERVER; + std::string err_msg; + + virtual bool is_error() override { return true; } + + virtual json to_json() override { return format_error_response(err_msg, err_type); } +}; + +struct server_task_result_metrics : server_task_result { + int n_idle_slots; + int n_processing_slots; + int n_tasks_deferred; + int64_t t_start; + + int32_t kv_cache_tokens_count; + int32_t kv_cache_used_cells; + + // TODO: somehow reuse server_metrics in the future, instead of duplicating the fields + uint64_t n_prompt_tokens_processed_total = 0; + uint64_t t_prompt_processing_total = 0; + uint64_t n_tokens_predicted_total = 0; + uint64_t t_tokens_generation_total = 0; + + uint64_t n_prompt_tokens_processed = 0; + uint64_t t_prompt_processing = 0; + + uint64_t n_tokens_predicted = 0; + uint64_t t_tokens_generation = 0; + + uint64_t n_decode_total = 0; + uint64_t n_busy_slots_total = 0; + + // while we can also use std::vector this requires copying the slot object which can be quite messy + // therefore, we use json to temporarily store the slot.to_json() result + json slots_data = json::array(); + + virtual json to_json() override { + return json{ + {"idle", n_idle_slots}, + {"processing", n_processing_slots}, + {"deferred", n_tasks_deferred}, + {"t_start", t_start}, + + {"n_prompt_tokens_processed_total", n_prompt_tokens_processed_total}, + {"t_tokens_generation_total", t_tokens_generation_total}, + {"n_tokens_predicted_total", n_tokens_predicted_total}, + {"t_prompt_processing_total", t_prompt_processing_total}, + + {"n_prompt_tokens_processed", n_prompt_tokens_processed}, + {"t_prompt_processing", t_prompt_processing}, + {"n_tokens_predicted", n_tokens_predicted}, + {"t_tokens_generation", t_tokens_generation}, + + {"n_decode_total", n_decode_total}, + {"n_busy_slots_total", n_busy_slots_total}, + + {"kv_cache_tokens_count", kv_cache_tokens_count}, + {"kv_cache_used_cells", kv_cache_used_cells}, + + {"slots", slots_data}, + }; + } +}; + +struct server_task_result_slot_save_load : server_task_result { + std::string filename; + bool is_save; // true = save, false = load + + size_t n_tokens; + size_t n_bytes; + double t_ms; + + virtual json to_json() override { + if (is_save) { + return json{ + {"id_slot", id_slot}, {"filename", filename}, {"n_saved", n_tokens}, + {"n_written", n_bytes}, {"timings", {{"save_ms", t_ms}}}, + }; + } else { + return json{ + {"id_slot", id_slot}, + {"filename", filename}, + {"n_restored", n_tokens}, + {"n_read", n_bytes}, + {"timings", {{"restore_ms", t_ms}}}, + }; + } + } +}; + +struct server_task_result_slot_erase : server_task_result { + size_t n_erased; + + virtual json to_json() override { + return json{ + {"id_slot", id_slot}, + {"n_erased", n_erased}, + }; + } +}; + +struct server_task_result_apply_lora : server_task_result { + virtual json to_json() override { return json{{"success", true}}; } +}; + +struct server_slot { + int id; + int id_task = -1; + + // only used for completion/embedding/infill/rerank + server_task_type task_type = SERVER_TASK_TYPE_COMPLETION; + + llama_batch batch_spec = {}; + + llama_context *ctx = nullptr; + llama_context *ctx_dft = nullptr; + + common_speculative *spec = nullptr; + + std::vector lora; + + // the index relative to completion multi-task request + size_t index = 0; + + struct slot_params params; + + slot_state state = SLOT_STATE_IDLE; + + // used to determine the slot that has been used the longest + int64_t t_last_used = -1; + + // generation props + int32_t n_ctx = 0; // context size per slot + int32_t n_past = 0; + int32_t n_decoded = 0; + int32_t n_remaining = -1; + int32_t i_batch = -1; + int32_t n_predict = -1; // TODO: disambiguate from params.n_predict + + // n_prompt_tokens may not be equal to prompt_tokens.size(), because prompt maybe truncated + int32_t n_prompt_tokens = 0; + int32_t n_prompt_tokens_processed = 0; + + // input prompt tokens + llama_tokens prompt_tokens; + + size_t last_nl_pos = 0; + + std::string generated_text; + llama_tokens generated_tokens; + + llama_tokens cache_tokens; + + std::vector generated_token_probs; + + bool has_next_token = true; + bool has_new_line = false; + bool truncated = false; + stop_type stop; + + std::string stopping_word; + + // sampling + json json_schema; + + struct common_sampler *smpl = nullptr; + + llama_token sampled; + + common_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY; + + // stats + size_t n_sent_text = 0; // number of sent text character + + int64_t t_start_process_prompt; + int64_t t_start_generation; + + double t_prompt_processing; // ms + double t_token_generation; // ms + + std::function callback_on_release; + + void reset() { + SLT_DBG(*this, "%s", "\n"); + + n_prompt_tokens = 0; + last_nl_pos = 0; + generated_text = ""; + has_new_line = false; + truncated = false; + stop = STOP_TYPE_NONE; + stopping_word = ""; + n_past = 0; + n_sent_text = 0; + task_type = SERVER_TASK_TYPE_COMPLETION; + + generated_tokens.clear(); + generated_token_probs.clear(); + } + + bool is_non_causal() const { + return task_type == SERVER_TASK_TYPE_EMBEDDING || task_type == SERVER_TASK_TYPE_RERANK; + } + + bool can_batch_with(server_slot &other_slot) { + return is_non_causal() == other_slot.is_non_causal() && are_lora_equal(lora, other_slot.lora); + } + + bool has_budget(const common_params &global_params) { + if (params.n_predict == -1 && global_params.n_predict == -1) { + return true; // limitless + } + + n_remaining = -1; + + if (params.n_predict != -1) { + n_remaining = params.n_predict - n_decoded; + } else if (global_params.n_predict != -1) { + n_remaining = global_params.n_predict - n_decoded; + } + + return n_remaining > 0; // no budget + } + + bool is_processing() const { return state != SLOT_STATE_IDLE; } + + bool can_speculate() const { return ctx_dft && params.speculative.n_max > 0 && params.cache_prompt; } + + void add_token(const completion_token_output &token) { + if (!is_processing()) { + SLT_WRN(*this, "%s", "slot is not processing\n"); + return; + } + generated_token_probs.push_back(token); + } + + void release() { + if (is_processing()) { + SLT_INF(*this, "stop processing: n_past = %d, truncated = %d\n", n_past, truncated); + + t_last_used = ggml_time_us(); + t_token_generation = (ggml_time_us() - t_start_generation) / 1e3; + state = SLOT_STATE_IDLE; + callback_on_release(id); + } + } + + result_timings get_timings() const { + result_timings timings; + timings.prompt_n = n_prompt_tokens_processed; + timings.prompt_ms = t_prompt_processing; + timings.prompt_per_token_ms = t_prompt_processing / n_prompt_tokens_processed; + timings.prompt_per_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed; + + timings.predicted_n = n_decoded; + timings.predicted_ms = t_token_generation; + timings.predicted_per_token_ms = t_token_generation / n_decoded; + timings.predicted_per_second = 1e3 / t_token_generation * n_decoded; + + return timings; + } + + size_t find_stopping_strings(const std::string &text, const size_t last_token_size, bool is_full_stop) { + size_t stop_pos = std::string::npos; + + for (const std::string &word : params.antiprompt) { + size_t pos; + + if (is_full_stop) { + const size_t tmp = word.size() + last_token_size; + const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0; + + pos = text.find(word, from_pos); + } else { + // otherwise, partial stop + pos = find_partial_stop_string(word, text); + } + + if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) { + if (is_full_stop) { + stop = STOP_TYPE_WORD; + stopping_word = word; + has_next_token = false; + } + stop_pos = pos; + } + } + + return stop_pos; + } + + void print_timings() const { + const double t_prompt = t_prompt_processing / n_prompt_tokens_processed; + const double n_prompt_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed; + + const double t_gen = t_token_generation / n_decoded; + const double n_gen_second = 1e3 / t_token_generation * n_decoded; + + SLT_INF(*this, + "\n" + "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n" + " eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n" + " total time = %10.2f ms / %5d tokens\n", + t_prompt_processing, n_prompt_tokens_processed, t_prompt, n_prompt_second, t_token_generation, + n_decoded, t_gen, n_gen_second, t_prompt_processing + t_token_generation, + n_prompt_tokens_processed + n_decoded); + } + + json to_json() const { + return json{ + {"id", id}, + {"id_task", id_task}, + {"n_ctx", n_ctx}, + {"speculative", can_speculate()}, + {"is_processing", is_processing()}, + {"non_causal", is_non_causal()}, + {"params", params.to_json()}, + {"prompt", common_detokenize(ctx, prompt_tokens)}, + {"next_token", + { + {"has_next_token", has_next_token}, + {"has_new_line", has_new_line}, + {"n_remain", n_remaining}, + {"n_decoded", n_decoded}, + {"stopping_word", stopping_word}, + }}, + }; + } +}; + +struct server_metrics { + int64_t t_start = 0; + + uint64_t n_prompt_tokens_processed_total = 0; + uint64_t t_prompt_processing_total = 0; + uint64_t n_tokens_predicted_total = 0; + uint64_t t_tokens_generation_total = 0; + + uint64_t n_prompt_tokens_processed = 0; + uint64_t t_prompt_processing = 0; + + uint64_t n_tokens_predicted = 0; + uint64_t t_tokens_generation = 0; + + uint64_t n_decode_total = 0; + uint64_t n_busy_slots_total = 0; + + void init() { t_start = ggml_time_us(); } + + void on_prompt_eval(const server_slot &slot) { + n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed; + n_prompt_tokens_processed += slot.n_prompt_tokens_processed; + t_prompt_processing += slot.t_prompt_processing; + t_prompt_processing_total += slot.t_prompt_processing; + } + + void on_prediction(const server_slot &slot) { + n_tokens_predicted_total += slot.n_decoded; + n_tokens_predicted += slot.n_decoded; + t_tokens_generation += slot.t_token_generation; + t_tokens_generation_total += slot.t_token_generation; + } + + void on_decoded(const std::vector &slots) { + n_decode_total++; + for (const auto &slot : slots) { + if (slot.is_processing()) { + n_busy_slots_total++; + } + } + } + + void reset_bucket() { + n_prompt_tokens_processed = 0; + t_prompt_processing = 0; + n_tokens_predicted = 0; + t_tokens_generation = 0; + } +}; + +struct server_queue { + int id = 0; + bool running; + + // queues + std::deque queue_tasks; + std::deque queue_tasks_deferred; + + std::mutex mutex_tasks; + std::condition_variable condition_tasks; + + // callback functions + std::function callback_new_task; + std::function callback_update_slots; + + // Add a new task to the end of the queue + int post(server_task task, bool front = false) { + std::unique_lock lock(mutex_tasks); + GGML_ASSERT(task.id != -1); + // if this is cancel task make sure to clean up pending tasks + if (task.type == SERVER_TASK_TYPE_CANCEL) { + cleanup_pending_task(task.id_target); + } + QUE_DBG("new task, id = %d, front = %d\n", task.id, front); + if (front) { + queue_tasks.push_front(std::move(task)); + } else { + queue_tasks.push_back(std::move(task)); + } + condition_tasks.notify_one(); + return task.id; + } + + // multi-task version of post() + int post(std::vector &tasks, bool front = false) { + std::unique_lock lock(mutex_tasks); + for (auto &task : tasks) { + if (task.id == -1) { + task.id = id++; + } + // if this is cancel task make sure to clean up pending tasks + if (task.type == SERVER_TASK_TYPE_CANCEL) { + cleanup_pending_task(task.id_target); + } + QUE_DBG("new task, id = %d/%d, front = %d\n", task.id, (int)tasks.size(), front); + if (front) { + queue_tasks.push_front(std::move(task)); + } else { + queue_tasks.push_back(std::move(task)); + } + } + condition_tasks.notify_one(); + return 0; + } + + // Add a new task, but defer until one slot is available + void defer(server_task task) { + std::unique_lock lock(mutex_tasks); + QUE_DBG("defer task, id = %d\n", task.id); + queue_tasks_deferred.push_back(std::move(task)); + condition_tasks.notify_one(); + } + + // Get the next id for creating a new task + int get_new_id() { + std::unique_lock lock(mutex_tasks); + int new_id = id++; + return new_id; + } + + // Register function to process a new task + void on_new_task(std::function callback) { callback_new_task = std::move(callback); } + + // Register the function to be called when all slots data is ready to be processed + void on_update_slots(std::function callback) { callback_update_slots = std::move(callback); } + + // Call when the state of one slot is changed, it will move one task from deferred to main queue + void pop_deferred_task() { + std::unique_lock lock(mutex_tasks); + if (!queue_tasks_deferred.empty()) { + queue_tasks.emplace_back(std::move(queue_tasks_deferred.front())); + queue_tasks_deferred.pop_front(); + } + condition_tasks.notify_one(); + } + + // end the start_loop routine + void terminate() { + std::unique_lock lock(mutex_tasks); + running = false; + condition_tasks.notify_all(); + } + + /** + * Main loop consists of these steps: + * - Wait until a new task arrives + * - Process the task (i.e. maybe copy data into slot) + * - Check if multitask is finished + * - Update all slots + */ + void start_loop() { + running = true; + + while (true) { + QUE_DBG("%s", "processing new tasks\n"); + + while (true) { + std::unique_lock lock(mutex_tasks); + if (!running) { + QUE_DBG("%s", "terminate\n"); + return; + } + if (queue_tasks.empty()) { + lock.unlock(); + break; + } + server_task task = queue_tasks.front(); + queue_tasks.pop_front(); + lock.unlock(); + + QUE_DBG("processing task, id = %d\n", task.id); + callback_new_task(std::move(task)); + } + + // all tasks in the current loop is processed, slots data is now ready + QUE_DBG("%s", "update slots\n"); + + callback_update_slots(); + + QUE_DBG("%s", "waiting for new tasks\n"); + { + std::unique_lock lock(mutex_tasks); + if (!running) { + QUE_DBG("%s", "terminate\n"); + return; + } + if (queue_tasks.empty()) { + condition_tasks.wait(lock, [&] { return (!queue_tasks.empty() || !running); }); + } + } + } + } + + private: + void cleanup_pending_task(int id_target) { + // no need lock because this is called exclusively by post() + auto rm_func = [id_target](const server_task &task) { return task.id_target == id_target; }; + queue_tasks.erase(std::remove_if(queue_tasks.begin(), queue_tasks.end(), rm_func), queue_tasks.end()); + queue_tasks_deferred.erase(std::remove_if(queue_tasks_deferred.begin(), queue_tasks_deferred.end(), rm_func), + queue_tasks_deferred.end()); + } +}; + +struct server_response { + // for keeping track of all tasks waiting for the result + std::unordered_set waiting_task_ids; + + // the main result queue (using ptr for polymorphism) + std::vector queue_results; + + std::mutex mutex_results; + std::condition_variable condition_results; + + // add the id_task to the list of tasks waiting for response + void add_waiting_task_id(int id_task) { + SRV_DBG("add task %d to waiting list. current waiting = %d (before add)\n", id_task, + (int)waiting_task_ids.size()); + + std::unique_lock lock(mutex_results); + waiting_task_ids.insert(id_task); + } + + void add_waiting_tasks(const std::vector &tasks) { + std::unique_lock lock(mutex_results); + + for (const auto &task : tasks) { + SRV_DBG("add task %d to waiting list. current waiting = %d (before add)\n", task.id, + (int)waiting_task_ids.size()); + waiting_task_ids.insert(task.id); + } + } + + // when the request is finished, we can remove task associated with it + void remove_waiting_task_id(int id_task) { + SRV_DBG("remove task %d from waiting list. current waiting = %d (before remove)\n", id_task, + (int)waiting_task_ids.size()); + + std::unique_lock lock(mutex_results); + waiting_task_ids.erase(id_task); + // make sure to clean up all pending results + queue_results.erase(std::remove_if(queue_results.begin(), queue_results.end(), + [id_task](const server_task_result_ptr &res) { return res->id == id_task; }), + queue_results.end()); + } + + void remove_waiting_task_ids(const std::unordered_set &id_tasks) { + std::unique_lock lock(mutex_results); + + for (const auto &id_task : id_tasks) { + SRV_DBG("remove task %d from waiting list. current waiting = %d (before remove)\n", id_task, + (int)waiting_task_ids.size()); + waiting_task_ids.erase(id_task); + } + } + + // This function blocks the thread until there is a response for one of the id_tasks + server_task_result_ptr recv(const std::unordered_set &id_tasks) { + while (true) { + std::unique_lock lock(mutex_results); + condition_results.wait(lock, [&] { return !queue_results.empty(); }); + + for (size_t i = 0; i < queue_results.size(); i++) { + if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) { + server_task_result_ptr res = std::move(queue_results[i]); + queue_results.erase(queue_results.begin() + i); + return res; + } + } + } + + // should never reach here + } + + // same as recv(), but have timeout in seconds + // if timeout is reached, nullptr is returned + server_task_result_ptr recv_with_timeout(const std::unordered_set &id_tasks, int timeout) { + while (true) { + std::unique_lock lock(mutex_results); + + for (int i = 0; i < (int)queue_results.size(); i++) { + if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) { + server_task_result_ptr res = std::move(queue_results[i]); + queue_results.erase(queue_results.begin() + i); + return res; + } + } + + std::cv_status cr_res = condition_results.wait_for(lock, std::chrono::seconds(timeout)); + if (cr_res == std::cv_status::timeout) { + return nullptr; + } + } + + // should never reach here + } + + // single-task version of recv() + server_task_result_ptr recv(int id_task) { + std::unordered_set id_tasks = {id_task}; + return recv(id_tasks); + } + + // Send a new result to a waiting id_task + void send(server_task_result_ptr &&result) { + SRV_DBG("sending result for task id = %d\n", result->id); + + std::unique_lock lock(mutex_results); + for (const auto &id_task : waiting_task_ids) { + if (result->id == id_task) { + SRV_DBG("task id = %d pushed to result queue\n", result->id); + + queue_results.emplace_back(std::move(result)); + condition_results.notify_all(); + return; + } + } + } +}; + +struct server_context { + common_params params_base; + + // note: keep these alive - they determine the lifetime of the model, context, etc. + common_init_result llama_init; + common_init_result llama_init_dft; + + llama_model *model = nullptr; + llama_context *ctx = nullptr; + + const llama_vocab *vocab = nullptr; + + llama_model *model_dft = nullptr; + + llama_context_params cparams_dft; + + llama_batch batch = {}; + + bool clean_kv_cache = true; + bool add_bos_token = true; + bool has_eos_token = false; + + int32_t n_ctx; // total context for all clients / slots + + // slots / clients + std::vector slots; + json default_generation_settings_for_props; + + server_queue queue_tasks; + server_response queue_results; + + server_metrics metrics; + + // Necessary similarity of prompt for slot selection + float slot_prompt_similarity = 0.0f; + + common_chat_templates_ptr chat_templates; + + ~server_context() { + // Clear any sampling context + for (server_slot &slot : slots) { + common_sampler_free(slot.smpl); + slot.smpl = nullptr; + + llama_free(slot.ctx_dft); + slot.ctx_dft = nullptr; + + common_speculative_free(slot.spec); + slot.spec = nullptr; + + llama_batch_free(slot.batch_spec); + } + + llama_batch_free(batch); + } + + bool load_model(const common_params ¶ms) { + SRV_INF("loading model '%s'\n", params.model.c_str()); + + params_base = params; + + llama_init = common_init_from_params(params_base); + + model = llama_init.model.get(); + ctx = llama_init.context.get(); + + if (model == nullptr) { + SRV_ERR("failed to load model, '%s'\n", params_base.model.c_str()); + return false; + } + + vocab = llama_model_get_vocab(model); + + n_ctx = llama_n_ctx(ctx); + + add_bos_token = llama_vocab_get_add_bos(vocab); + has_eos_token = llama_vocab_eos(vocab) != LLAMA_TOKEN_NULL; + + if (!params_base.speculative.model.empty() || !params_base.speculative.hf_repo.empty()) { + SRV_INF("loading draft model '%s'\n", params_base.speculative.model.c_str()); + + auto params_dft = params_base; + + params_dft.devices = params_base.speculative.devices; + params_dft.hf_file = params_base.speculative.hf_file; + params_dft.hf_repo = params_base.speculative.hf_repo; + params_dft.model = params_base.speculative.model; + params_dft.model_url = params_base.speculative.model_url; + params_dft.n_ctx = params_base.speculative.n_ctx == 0 ? params_base.n_ctx / params_base.n_parallel + : params_base.speculative.n_ctx; + params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers; + params_dft.n_parallel = 1; + + llama_init_dft = common_init_from_params(params_dft); + + model_dft = llama_init_dft.model.get(); + + if (model_dft == nullptr) { + SRV_ERR("failed to load draft model, '%s'\n", params_base.speculative.model.c_str()); + return false; + } + + if (!common_speculative_are_compatible(ctx, llama_init_dft.context.get())) { + SRV_ERR("the draft model '%s' is not compatible with the target model '%s'\n", + params_base.speculative.model.c_str(), params_base.model.c_str()); + + return false; + } + + const int n_ctx_dft = llama_n_ctx(llama_init_dft.context.get()); + + cparams_dft = common_context_params_to_llama(params_dft); + cparams_dft.n_batch = n_ctx_dft; + + // force F16 KV cache for the draft model for extra performance + cparams_dft.type_k = GGML_TYPE_F16; + cparams_dft.type_v = GGML_TYPE_F16; + + // the context is not needed - we will create one for each slot + llama_init_dft.context.reset(); + } + + chat_templates = common_chat_templates_init(model, params_base.chat_template); + try { + common_chat_format_example(chat_templates.get(), params.use_jinja); + } catch (const std::exception &e) { + SRV_WRN("%s: The chat template that comes with this model is not yet supported, falling back to chatml. " + "This may cause the model to output suboptimal responses\n", + __func__); + chat_templates = common_chat_templates_init(model, "chatml"); + } + + return true; + } + + void init() { + const int32_t n_ctx_slot = n_ctx / params_base.n_parallel; + + SRV_INF("initializing slots, n_slots = %d\n", params_base.n_parallel); + + for (int i = 0; i < params_base.n_parallel; i++) { + server_slot slot; + + slot.id = i; + slot.ctx = ctx; + slot.n_ctx = n_ctx_slot; + slot.n_predict = params_base.n_predict; + + if (model_dft) { + slot.batch_spec = llama_batch_init(params_base.speculative.n_max + 1, 0, 1); + + slot.ctx_dft = llama_init_from_model(model_dft, cparams_dft); + if (slot.ctx_dft == nullptr) { + SRV_ERR("%s", "failed to create draft context\n"); + return; + } + + slot.spec = common_speculative_init(slot.ctx_dft); + if (slot.spec == nullptr) { + SRV_ERR("%s", "failed to create speculator\n"); + return; + } + } + + SLT_INF(slot, "new slot n_ctx_slot = %d\n", slot.n_ctx); + + slot.params.sampling = params_base.sampling; + + slot.callback_on_release = [this](int) { queue_tasks.pop_deferred_task(); }; + + slot.reset(); + + slots.push_back(slot); + } + + default_generation_settings_for_props = slots[0].to_json(); + + // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens + // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not + // used) + { + const int32_t n_batch = llama_n_batch(ctx); + + // only a single seq_id per token is needed + batch = llama_batch_init(std::max(n_batch, params_base.n_parallel), 0, 1); + } + + metrics.init(); + } + + server_slot *get_slot_by_id(int id) { + for (server_slot &slot : slots) { + if (slot.id == id) { + return &slot; + } + } + + return nullptr; + } + + server_slot *get_available_slot(const server_task &task) { + server_slot *ret = nullptr; + + // find the slot that has at least n% prompt similarity + if (ret == nullptr && slot_prompt_similarity != 0.0f) { + int lcs_len = 0; + float similarity = 0; + + for (server_slot &slot : slots) { + // skip the slot if it is not available + if (slot.is_processing()) { + continue; + } + + // skip the slot if it does not contains cached tokens + if (slot.cache_tokens.empty()) { + continue; + } + + // length of the Longest Common Subsequence between the current slot's prompt and the input prompt + int cur_lcs_len = common_lcs(slot.cache_tokens, task.prompt_tokens); + + // fraction of the common subsequence length compared to the current slot's prompt length + float cur_similarity = static_cast(cur_lcs_len) / static_cast(slot.cache_tokens.size()); + + // select the current slot if the criteria match + if (cur_lcs_len > lcs_len && cur_similarity > slot_prompt_similarity) { + lcs_len = cur_lcs_len; + similarity = cur_similarity; + ret = &slot; + } + } + + if (ret != nullptr) { + SLT_DBG(*ret, "selected slot by lcs similarity, lcs_len = %d, similarity = %f\n", lcs_len, similarity); + } + } + + // find the slot that has been least recently used + if (ret == nullptr) { + int64_t t_last = ggml_time_us(); + for (server_slot &slot : slots) { + // skip the slot if it is not available + if (slot.is_processing()) { + continue; + } + + // select the current slot if the criteria match + if (slot.t_last_used < t_last) { + t_last = slot.t_last_used; + ret = &slot; + } + } + + if (ret != nullptr) { + SLT_DBG(*ret, "selected slot by lru, t_last = %" PRId64 "\n", t_last); + } + } + + return ret; + } + + bool launch_slot_with_task(server_slot &slot, const server_task &task) { + slot.reset(); + slot.id_task = task.id; + slot.index = task.index; + slot.task_type = task.type; + slot.params = std::move(task.params); + slot.prompt_tokens = std::move(task.prompt_tokens); + + if (!are_lora_equal(task.params.lora, slot.lora)) { + // if lora is changed, we cannot reuse cached tokens + slot.cache_tokens.clear(); + slot.lora = task.params.lora; + } + + SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str()); + + if (slot.n_predict > 0 && slot.params.n_predict > slot.n_predict) { + // Might be better to reject the request with a 400 ? + SLT_WRN(slot, "n_predict = %d exceeds server configuration, setting to %d\n", slot.params.n_predict, + slot.n_predict); + slot.params.n_predict = slot.n_predict; + } + + if (slot.params.ignore_eos && has_eos_token) { + slot.params.sampling.logit_bias.push_back({llama_vocab_eos(vocab), -INFINITY}); + } + + { + if (slot.smpl != nullptr) { + common_sampler_free(slot.smpl); + } + + slot.smpl = common_sampler_init(model, slot.params.sampling); + if (slot.smpl == nullptr) { + // for now, the only error that may happen here is invalid grammar + send_error(task, "Failed to parse grammar", ERROR_TYPE_INVALID_REQUEST); + return false; + } + } + + if (slot.ctx_dft) { + llama_batch_free(slot.batch_spec); + + slot.batch_spec = llama_batch_init(slot.params.speculative.n_max + 1, 0, 1); + } + + slot.state = SLOT_STATE_STARTED; + + SLT_INF(slot, "%s", "processing task\n"); + + return true; + } + + void kv_cache_clear() { + SRV_DBG("%s", "clearing KV cache\n"); + + // clear the entire KV cache + llama_kv_cache_clear(ctx); + clean_kv_cache = false; + } + + bool process_token(completion_token_output &result, server_slot &slot) { + // remember which tokens were sampled - used for repetition penalties during sampling + const std::string token_str = result.text_to_send; + slot.sampled = result.tok; + + slot.generated_text += token_str; + if (slot.params.return_tokens) { + slot.generated_tokens.push_back(result.tok); + } + slot.has_next_token = true; + + // check if there is incomplete UTF-8 character at the end + bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size(); + + // search stop word and delete it + if (!incomplete) { + size_t pos = std::min(slot.n_sent_text, slot.generated_text.size()); + + const std::string str_test = slot.generated_text.substr(pos); + bool send_text = true; + + size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true); + if (stop_pos != std::string::npos) { + slot.generated_text.erase(slot.generated_text.begin() + pos + stop_pos, slot.generated_text.end()); + pos = std::min(slot.n_sent_text, slot.generated_text.size()); + } else if (slot.has_next_token) { + stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false); + send_text = stop_pos == std::string::npos; + } + + // check if there is any token to predict + if (send_text) { + // no send the stop word in the response + result.text_to_send = slot.generated_text.substr(pos, std::string::npos); + slot.n_sent_text += result.text_to_send.size(); + // add the token to slot queue and cache + } else { + result.text_to_send = ""; + } + + slot.add_token(result); + if (slot.params.stream) { + send_partial_response(slot, result); + } + } + + if (incomplete) { + slot.has_next_token = true; + } + + // check the limits + if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params_base)) { + slot.stop = STOP_TYPE_LIMIT; + slot.has_next_token = false; + + SLT_DBG(slot, "stopped by limit, n_decoded = %d, n_predict = %d\n", slot.n_decoded, slot.params.n_predict); + } + + if (slot.has_new_line) { + // if we have already seen a new line, we stop after a certain time limit + if (slot.params.t_max_predict_ms > 0 && + (ggml_time_us() - slot.t_start_generation > 1000.0f * slot.params.t_max_predict_ms)) { + slot.stop = STOP_TYPE_LIMIT; + slot.has_next_token = false; + + SLT_DBG(slot, "stopped by time limit, n_decoded = %d, t_max_predict_ms = %d ms\n", slot.n_decoded, + (int)slot.params.t_max_predict_ms); + } + + // require that each new line has a whitespace prefix (i.e. indentation) of at least slot.params.n_indent + if (slot.params.n_indent > 0) { + // check the current indentation + // TODO: improve by not doing it more than once for each new line + if (slot.last_nl_pos > 0) { + size_t pos = slot.last_nl_pos; + + int n_indent = 0; + while (pos < slot.generated_text.size() && + (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) { + n_indent++; + pos++; + } + + if (pos < slot.generated_text.size() && n_indent < slot.params.n_indent) { + slot.stop = STOP_TYPE_LIMIT; + slot.has_next_token = false; + + // cut the last line + slot.generated_text.erase(pos, std::string::npos); + + SLT_DBG(slot, "stopped by indentation limit, n_decoded = %d, n_indent = %d\n", slot.n_decoded, + n_indent); + } + } + + // find the next new line + { + const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos); + + if (pos != std::string::npos) { + slot.last_nl_pos = pos + 1; + } + } + } + } + + // check if there is a new line in the generated text + if (result.text_to_send.find('\n') != std::string::npos) { + slot.has_new_line = true; + } + + // if context shift is disabled, we stop when it reaches the context limit + if (slot.n_past >= slot.n_ctx) { + slot.truncated = true; + slot.stop = STOP_TYPE_LIMIT; + slot.has_next_token = false; + + SLT_DBG(slot, + "stopped due to running out of context capacity, n_past = %d, n_prompt_tokens = %d, n_decoded = " + "%d, n_ctx = %d\n", + slot.n_decoded, slot.n_prompt_tokens, slot.n_past, slot.n_ctx); + } + + if (llama_vocab_is_eog(vocab, result.tok)) { + slot.stop = STOP_TYPE_EOS; + slot.has_next_token = false; + + SLT_DBG(slot, "%s", "stopped by EOS\n"); + } + + const auto n_ctx_train = llama_model_n_ctx_train(model); + + if (slot.params.n_predict < 1 && slot.n_predict < 1 && slot.n_prompt_tokens + slot.n_decoded >= n_ctx_train) { + slot.truncated = true; + slot.stop = STOP_TYPE_LIMIT; + slot.has_next_token = false; // stop prediction + + SLT_WRN(slot, + "n_predict (%d) is set for infinite generation. " + "Limiting generated tokens to n_ctx_train (%d) to avoid EOS-less generation infinite loop\n", + slot.params.n_predict, n_ctx_train); + } + + SLT_DBG(slot, "n_decoded = %d, n_remaining = %d, next token: %5d '%s'\n", slot.n_decoded, slot.n_remaining, + result.tok, token_str.c_str()); + + return slot.has_next_token; // continue + } + + void populate_token_probs(const server_slot &slot, completion_token_output &result, bool post_sampling, + bool special, int idx) { + size_t n_probs = slot.params.sampling.n_probs; + size_t n_vocab = llama_vocab_n_tokens(vocab); + if (post_sampling) { + const auto *cur_p = common_sampler_get_candidates(slot.smpl); + const size_t max_probs = cur_p->size; + + // set probability for sampled token + for (size_t i = 0; i < max_probs; i++) { + if (cur_p->data[i].id == result.tok) { + result.prob = cur_p->data[i].p; + break; + } + } + + // set probability for top n_probs tokens + result.probs.reserve(max_probs); + for (size_t i = 0; i < std::min(max_probs, n_probs); i++) { + result.probs.push_back( + {cur_p->data[i].id, common_token_to_piece(ctx, cur_p->data[i].id, special), cur_p->data[i].p}); + } + } else { + // TODO: optimize this with min-p optimization + std::vector cur = get_token_probabilities(ctx, idx); + + // set probability for sampled token + for (size_t i = 0; i < n_vocab; i++) { + // set probability for sampled token + if (cur[i].id == result.tok) { + result.prob = cur[i].p; + break; + } + } + + // set probability for top n_probs tokens + result.probs.reserve(n_probs); + for (size_t i = 0; i < std::min(n_vocab, n_probs); i++) { + result.probs.push_back({cur[i].id, common_token_to_piece(ctx, cur[i].id, special), cur[i].p}); + } + } + } + + void send_error(const server_task &task, const std::string &error, const enum error_type type = ERROR_TYPE_SERVER) { + send_error(task.id, error, type); + } + + void send_error(const server_slot &slot, const std::string &error, const enum error_type type = ERROR_TYPE_SERVER) { + send_error(slot.id_task, error, type); + } + + void send_error(const int id_task, const std::string &error, const enum error_type type = ERROR_TYPE_SERVER) { + SRV_ERR("task id = %d, error: %s\n", id_task, error.c_str()); + + auto res = std::make_unique(); + res->id = id_task; + res->err_type = type; + res->err_msg = error; + + queue_results.send(std::move(res)); + } + + void send_partial_response(server_slot &slot, const completion_token_output &tkn) { + auto res = std::make_unique(); + + res->id = slot.id_task; + res->index = slot.index; + res->content = tkn.text_to_send; + res->tokens = {tkn.tok}; + + res->n_decoded = slot.n_decoded; + res->n_prompt_tokens = slot.n_prompt_tokens; + res->post_sampling_probs = slot.params.post_sampling_probs; + + res->verbose = slot.params.verbose; + res->oaicompat = slot.params.oaicompat; + res->oaicompat_model = slot.params.oaicompat_model; + res->oaicompat_cmpl_id = slot.params.oaicompat_cmpl_id; + + // populate res.probs_output + if (slot.params.sampling.n_probs > 0) { + res->prob_output = tkn; // copy the token probs + } + + // populate timings if this is final response or timings_per_token is enabled + if (slot.stop != STOP_TYPE_NONE || slot.params.timings_per_token) { + res->timings = slot.get_timings(); + } + + queue_results.send(std::move(res)); + } + + void send_final_response(server_slot &slot) { + auto res = std::make_unique(); + res->id = slot.id_task; + res->id_slot = slot.id; + + res->index = slot.index; + res->content = std::move(slot.generated_text); + res->tokens = std::move(slot.generated_tokens); + res->timings = slot.get_timings(); + res->prompt = common_detokenize(ctx, slot.prompt_tokens, true); + res->response_fields = std::move(slot.params.response_fields); + + res->truncated = slot.truncated; + res->n_decoded = slot.n_decoded; + res->n_prompt_tokens = slot.n_prompt_tokens; + res->n_tokens_cached = slot.n_past; + res->has_new_line = slot.has_new_line; + res->stopping_word = slot.stopping_word; + res->stop = slot.stop; + res->post_sampling_probs = slot.params.post_sampling_probs; + + res->verbose = slot.params.verbose; + res->stream = slot.params.stream; + res->oaicompat = slot.params.oaicompat; + res->oaicompat_model = slot.params.oaicompat_model; + res->oaicompat_cmpl_id = slot.params.oaicompat_cmpl_id; + res->oaicompat_chat_format = slot.params.oaicompat_chat_format; + // populate res.probs_output + if (slot.params.sampling.n_probs > 0) { + if (!slot.params.stream && slot.stop == STOP_TYPE_WORD) { + const llama_tokens stop_word_toks = common_tokenize(ctx, slot.stopping_word, false); + + size_t safe_offset = std::min(slot.generated_token_probs.size(), stop_word_toks.size()); + res->probs_output = std::vector( + slot.generated_token_probs.begin(), slot.generated_token_probs.end() - safe_offset); + } else { + res->probs_output = std::vector(slot.generated_token_probs.begin(), + slot.generated_token_probs.end()); + } + } + + res->generation_params = slot.params; // copy the parameters + + queue_results.send(std::move(res)); + } + + void send_embedding(const server_slot &slot, const llama_batch &batch) { + auto res = std::make_unique(); + res->id = slot.id_task; + res->index = slot.index; + res->n_tokens = slot.n_prompt_tokens; + res->oaicompat = slot.params.oaicompat; + + const int n_embd = llama_model_n_embd(model); + + std::vector embd_res(n_embd, 0.0f); + + for (int i = 0; i < batch.n_tokens; ++i) { + if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) { + continue; + } + + const float *embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); + if (embd == NULL) { + embd = llama_get_embeddings_ith(ctx, i); + } + + if (embd == NULL) { + SLT_ERR(slot, "failed to get embeddings, token = %d, seq_id = %d\n", batch.token[i], + batch.seq_id[i][0]); + + res->embedding.push_back(std::vector(n_embd, 0.0f)); + continue; + } + + // normalize only when there is pooling + // TODO: configurable + if (llama_pooling_type(slot.ctx) != LLAMA_POOLING_TYPE_NONE) { + common_embd_normalize(embd, embd_res.data(), n_embd, 2); + res->embedding.push_back(embd_res); + } else { + res->embedding.push_back({embd, embd + n_embd}); + } + } + + SLT_DBG(slot, "%s", "sending embeddings\n"); + + queue_results.send(std::move(res)); + } + + void send_rerank(const server_slot &slot, const llama_batch &batch) { + auto res = std::make_unique(); + res->id = slot.id_task; + res->index = slot.index; + res->n_tokens = slot.n_prompt_tokens; + + for (int i = 0; i < batch.n_tokens; ++i) { + if (!batch.logits[i] || batch.seq_id[i][0] != slot.id) { + continue; + } + + const float *embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); + if (embd == NULL) { + embd = llama_get_embeddings_ith(ctx, i); + } + + if (embd == NULL) { + SLT_ERR(slot, "failed to get embeddings, token = %d, seq_id = %d\n", batch.token[i], + batch.seq_id[i][0]); + + res->score = -1e6; + continue; + } + + res->score = embd[0]; + } + + SLT_DBG(slot, "sending rerank result, res.score = %f\n", res->score); + + queue_results.send(std::move(res)); + } + + // + // Functions to create new task(s) and receive result(s) + // + + void cancel_tasks(const std::unordered_set &id_tasks) { + std::vector cancel_tasks; + cancel_tasks.reserve(id_tasks.size()); + for (const auto &id_task : id_tasks) { + SRV_WRN("cancel task, id_task = %d\n", id_task); + + server_task task(SERVER_TASK_TYPE_CANCEL); + task.id_target = id_task; + queue_results.remove_waiting_task_id(id_task); + cancel_tasks.push_back(task); + } + // push to beginning of the queue, so it has highest priority + queue_tasks.post(cancel_tasks, true); + } + + // receive the results from task(s) + void receive_multi_results(const std::unordered_set &id_tasks, + const std::function &)> &result_handler, + const std::function &error_handler, + const std::function &is_connection_closed) { + std::vector results(id_tasks.size()); + for (int i = 0; i < (int)id_tasks.size(); i++) { + server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, HTTP_POLLING_SECONDS); + + if (is_connection_closed()) { + cancel_tasks(id_tasks); + return; + } + + if (result == nullptr) { + i--; // retry + continue; + } + + if (result->is_error()) { + error_handler(result->to_json()); + cancel_tasks(id_tasks); + return; + } + + GGML_ASSERT(dynamic_cast(result.get()) != nullptr || + dynamic_cast(result.get()) != nullptr || + dynamic_cast(result.get()) != nullptr); + const size_t idx = result->get_index(); + GGML_ASSERT(idx < results.size() && "index out of range"); + results[idx] = std::move(result); + } + result_handler(results); + } + + // receive the results from task(s), in stream mode + void receive_cmpl_results_stream(const std::unordered_set &id_tasks, + const std::function &result_handler, + const std::function &error_handler, + const std::function &is_connection_closed) { + size_t n_finished = 0; + while (true) { + server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, HTTP_POLLING_SECONDS); + + if (is_connection_closed()) { + cancel_tasks(id_tasks); + return; + } + + if (result == nullptr) { + continue; // retry + } + + if (result->is_error()) { + error_handler(result->to_json()); + cancel_tasks(id_tasks); + return; + } + + GGML_ASSERT(dynamic_cast(result.get()) != nullptr || + dynamic_cast(result.get()) != nullptr); + if (!result_handler(result)) { + cancel_tasks(id_tasks); + break; + } + + if (result->is_stop()) { + if (++n_finished == id_tasks.size()) { + break; + } + } + } + } + + // + // Functions to process the task + // + + void process_single_task(server_task task) { + switch (task.type) { + case SERVER_TASK_TYPE_COMPLETION: + case SERVER_TASK_TYPE_INFILL: + case SERVER_TASK_TYPE_EMBEDDING: + case SERVER_TASK_TYPE_RERANK: { + const int id_slot = task.id_selected_slot; + + server_slot *slot = id_slot != -1 ? get_slot_by_id(id_slot) : get_available_slot(task); + + if (slot == nullptr) { + // if no slot is available, we defer this task for processing later + SRV_DBG("no slot is available, defer task, id_task = %d\n", task.id); + queue_tasks.defer(task); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(task); + break; + } + + if (!launch_slot_with_task(*slot, task)) { + SRV_ERR("failed to launch slot with task, id_task = %d\n", task.id); + break; + } + } break; + case SERVER_TASK_TYPE_CANCEL: { + // release slot linked with the task id + for (auto &slot : slots) { + if (slot.id_task == task.id_target) { + slot.release(); + break; + } + } + } break; + case SERVER_TASK_TYPE_NEXT_RESPONSE: { + // do nothing + } break; + case SERVER_TASK_TYPE_METRICS: { + json slots_data = json::array(); + + int n_idle_slots = 0; + int n_processing_slots = 0; + + for (server_slot &slot : slots) { + json slot_data = slot.to_json(); + + if (slot.is_processing()) { + n_processing_slots++; + } else { + n_idle_slots++; + } + + slots_data.push_back(slot_data); + } + SRV_DBG("n_idle_slots = %d, n_processing_slots = %d\n", n_idle_slots, n_processing_slots); + + auto res = std::make_unique(); + res->id = task.id; + res->slots_data = std::move(slots_data); + res->n_idle_slots = n_idle_slots; + res->n_processing_slots = n_processing_slots; + res->n_tasks_deferred = queue_tasks.queue_tasks_deferred.size(); + res->t_start = metrics.t_start; + + res->kv_cache_tokens_count = llama_get_kv_cache_token_count(ctx); + res->kv_cache_used_cells = llama_get_kv_cache_used_cells(ctx); + + res->n_prompt_tokens_processed_total = metrics.n_prompt_tokens_processed_total; + res->t_prompt_processing_total = metrics.t_prompt_processing_total; + res->n_tokens_predicted_total = metrics.n_tokens_predicted_total; + res->t_tokens_generation_total = metrics.t_tokens_generation_total; + + res->n_prompt_tokens_processed = metrics.n_prompt_tokens_processed; + res->t_prompt_processing = metrics.t_prompt_processing; + res->n_tokens_predicted = metrics.n_tokens_predicted; + res->t_tokens_generation = metrics.t_tokens_generation; + + res->n_decode_total = metrics.n_decode_total; + res->n_busy_slots_total = metrics.n_busy_slots_total; + + if (task.metrics_reset_bucket) { + metrics.reset_bucket(); + } + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_SAVE: { + int id_slot = task.slot_action.slot_id; + server_slot *slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(task); + break; + } + + const size_t token_count = slot->cache_tokens.size(); + const int64_t t_start = ggml_time_us(); + + std::string filename = task.slot_action.filename; + std::string filepath = task.slot_action.filepath; + + const size_t nwrite = + llama_state_seq_save_file(ctx, filepath.c_str(), slot->id, slot->cache_tokens.data(), token_count); + + const int64_t t_end = ggml_time_us(); + const double t_save_ms = (t_end - t_start) / 1000.0; + + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->filename = filename; + res->is_save = true; + res->n_tokens = token_count; + res->n_bytes = nwrite; + res->t_ms = t_save_ms; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_RESTORE: { + int id_slot = task.slot_action.slot_id; + server_slot *slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(task); + break; + } + + const int64_t t_start = ggml_time_us(); + + std::string filename = task.slot_action.filename; + std::string filepath = task.slot_action.filepath; + + slot->cache_tokens.resize(slot->n_ctx); + size_t token_count = 0; + size_t nread = llama_state_seq_load_file(ctx, filepath.c_str(), slot->id, slot->cache_tokens.data(), + slot->cache_tokens.size(), &token_count); + if (nread == 0) { + slot->cache_tokens.resize(0); + send_error(task, "Unable to restore slot, no available space in KV cache or invalid slot save file", + ERROR_TYPE_INVALID_REQUEST); + break; + } + slot->cache_tokens.resize(token_count); + + const int64_t t_end = ggml_time_us(); + const double t_restore_ms = (t_end - t_start) / 1000.0; + + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->filename = filename; + res->is_save = false; + res->n_tokens = token_count; + res->n_bytes = nread; + res->t_ms = t_restore_ms; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SLOT_ERASE: { + int id_slot = task.slot_action.slot_id; + server_slot *slot = get_slot_by_id(id_slot); + if (slot == nullptr) { + send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST); + break; + } + if (slot->is_processing()) { + // if requested slot is unavailable, we defer this task for processing later + SRV_DBG("requested slot is unavailable, defer task, id_task = %d\n", task.id); + queue_tasks.defer(task); + break; + } + + // Erase token cache + const size_t n_erased = slot->cache_tokens.size(); + llama_kv_cache_seq_rm(ctx, slot->id, -1, -1); + slot->cache_tokens.clear(); + + auto res = std::make_unique(); + res->id = task.id; + res->id_slot = id_slot; + res->n_erased = n_erased; + queue_results.send(std::move(res)); + } break; + case SERVER_TASK_TYPE_SET_LORA: { + params_base.lora_adapters = std::move(task.set_lora); + auto res = std::make_unique(); + res->id = task.id; + queue_results.send(std::move(res)); + } break; + } + } + + void update_slots() { + // check if all slots are idle + { + bool all_idle = true; + + for (auto &slot : slots) { + if (slot.is_processing()) { + all_idle = false; + break; + } + } + + if (all_idle) { + SRV_INF("%s", "all slots are idle\n"); + if (clean_kv_cache) { + kv_cache_clear(); + } + + return; + } + } + + { + SRV_DBG("%s", "posting NEXT_RESPONSE\n"); + + server_task task(SERVER_TASK_TYPE_NEXT_RESPONSE); + task.id = queue_tasks.get_new_id(); + queue_tasks.post(task); + } + + // apply context-shift if needed + // TODO: simplify and improve + for (server_slot &slot : slots) { + if (slot.is_processing() && slot.n_past + 1 >= slot.n_ctx) { + if (!params_base.ctx_shift) { + // this check is redundant (for good) + // we should never get here, because generation should already stopped in process_token() + slot.release(); + send_error(slot, "context shift is disabled", ERROR_TYPE_SERVER); + continue; + } + + // Shift context + const int n_keep = slot.params.n_keep + add_bos_token; + const int n_left = slot.n_past - n_keep; + const int n_discard = slot.params.n_discard ? slot.params.n_discard : (n_left / 2); + + SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, + n_discard); + + llama_kv_cache_seq_rm(ctx, slot.id, n_keep, n_keep + n_discard); + llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, slot.n_past, -n_discard); + + if (slot.params.cache_prompt) { + for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) { + slot.cache_tokens[i - n_discard] = slot.cache_tokens[i]; + } + + slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard); + } + + slot.n_past -= n_discard; + + slot.truncated = true; + } + } + + // start populating the batch for this iteration + common_batch_clear(batch); + + // track if given slot can be batched with slots already in the batch + server_slot *slot_batched = nullptr; + + auto accept_special_token = [&](server_slot &slot, llama_token token) { + return params_base.special || + slot.params.sampling.preserved_tokens.find(token) != slot.params.sampling.preserved_tokens.end(); + }; + + // frist, add sampled tokens from any ongoing sequences + for (auto &slot : slots) { + if (slot.state != SLOT_STATE_GENERATING) { + continue; + } + + // check if we can batch this slot with the previous one + if (!slot_batched) { + slot_batched = &slot; + } else if (!slot_batched->can_batch_with(slot)) { + continue; + } + + slot.i_batch = batch.n_tokens; + + common_batch_add(batch, slot.sampled, slot.n_past, {slot.id}, true); + + slot.n_past += 1; + + if (slot.params.cache_prompt) { + slot.cache_tokens.push_back(slot.sampled); + } + + SLT_DBG(slot, "slot decode token, n_ctx = %d, n_past = %d, n_cache_tokens = %d, truncated = %d\n", + slot.n_ctx, slot.n_past, (int)slot.cache_tokens.size(), slot.truncated); + } + + // process in chunks of params.n_batch + int32_t n_batch = llama_n_batch(ctx); + int32_t n_ubatch = llama_n_ubatch(ctx); + + // next, batch any pending prompts without exceeding n_batch + if (params_base.cont_batching || batch.n_tokens == 0) { + for (auto &slot : slots) { + // check if we can batch this slot with the previous one + if (slot.is_processing()) { + if (!slot_batched) { + slot_batched = &slot; + } else if (!slot_batched->can_batch_with(slot)) { + continue; + } + } + + // this slot still has a prompt to be processed + if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_STARTED) { + auto &prompt_tokens = slot.prompt_tokens; + + // TODO: maybe move branch to outside of this loop in the future + if (slot.state == SLOT_STATE_STARTED) { + slot.t_start_process_prompt = ggml_time_us(); + slot.t_start_generation = 0; + + slot.n_past = 0; + slot.n_prompt_tokens = prompt_tokens.size(); + slot.state = SLOT_STATE_PROCESSING_PROMPT; + + SLT_INF(slot, "new prompt, n_ctx_slot = %d, n_keep = %d, n_prompt_tokens = %d\n", slot.n_ctx, + slot.params.n_keep, slot.n_prompt_tokens); + + // print prompt tokens (for debugging) + if (1) { + // first 16 tokens (avoid flooding logs) + for (int i = 0; i < std::min(16, prompt_tokens.size()); i++) { + SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, prompt_tokens[i], + common_token_to_piece(ctx, prompt_tokens[i]).c_str()); + } + } else { + // all + for (int i = 0; i < (int)prompt_tokens.size(); i++) { + SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, prompt_tokens[i], + common_token_to_piece(ctx, prompt_tokens[i]).c_str()); + } + } + + // empty prompt passed -> release the slot and send empty response + if (prompt_tokens.empty()) { + SLT_WRN(slot, "%s", "empty prompt - releasing slot\n"); + + slot.release(); + slot.print_timings(); + send_final_response(slot); + continue; + } + + if (slot.is_non_causal()) { + if (slot.n_prompt_tokens > n_ubatch) { + slot.release(); + send_error(slot, "input is too large to process. increase the physical batch size", + ERROR_TYPE_SERVER); + continue; + } + + if (slot.n_prompt_tokens > slot.n_ctx) { + slot.release(); + send_error(slot, "input is larger than the max context size. skipping", + ERROR_TYPE_SERVER); + continue; + } + } else { + if (!params_base.ctx_shift) { + // if context shift is disabled, we make sure prompt size is smaller than KV size + // TODO: there should be a separate parameter that control prompt truncation + // context shift should be applied only during the generation phase + if (slot.n_prompt_tokens >= slot.n_ctx) { + slot.release(); + send_error(slot, + "the request exceeds the available context size. try increasing the " + "context size or enable context shift", + ERROR_TYPE_INVALID_REQUEST); + continue; + } + } + if (slot.params.n_keep < 0) { + slot.params.n_keep = slot.n_prompt_tokens; + } + slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep); + + // if input prompt is too big, truncate it + if (slot.n_prompt_tokens >= slot.n_ctx) { + const int n_left = slot.n_ctx - slot.params.n_keep; + + const int n_block_size = n_left / 2; + const int erased_blocks = + (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size; + + llama_tokens new_tokens(prompt_tokens.begin(), + prompt_tokens.begin() + slot.params.n_keep); + + new_tokens.insert(new_tokens.end(), + prompt_tokens.begin() + slot.params.n_keep + + erased_blocks * n_block_size, + prompt_tokens.end()); + + prompt_tokens = std::move(new_tokens); + + slot.truncated = true; + slot.n_prompt_tokens = prompt_tokens.size(); + + SLT_WRN(slot, + "input truncated, n_ctx = %d, n_keep = %d, n_left = %d, n_prompt_tokens = %d\n", + slot.n_ctx, slot.params.n_keep, n_left, slot.n_prompt_tokens); + + GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx); + } + + if (slot.params.cache_prompt) { + // reuse any previously computed tokens that are common with the new prompt + slot.n_past = common_lcp(slot.cache_tokens, prompt_tokens); + + // reuse chunks from the cached prompt by shifting their KV cache in the new position + if (params_base.n_cache_reuse > 0) { + size_t head_c = slot.n_past; // cache + size_t head_p = slot.n_past; // current prompt + + SLT_DBG(slot, "trying to reuse chunks with size > %d, slot.n_past = %d\n", + params_base.n_cache_reuse, slot.n_past); + + while (head_c < slot.cache_tokens.size() && head_p < prompt_tokens.size()) { + + size_t n_match = 0; + while (head_c + n_match < slot.cache_tokens.size() && + head_p + n_match < prompt_tokens.size() && + slot.cache_tokens[head_c + n_match] == prompt_tokens[head_p + n_match]) { + + n_match++; + } + + if (n_match >= (size_t)params_base.n_cache_reuse) { + SLT_INF(slot, + "reusing chunk with size %zu, shifting KV cache [%zu, %zu) -> " + "[%zu, %zu)\n", + n_match, head_c, head_c + n_match, head_p, head_p + n_match); + // for (size_t i = head_p; i < head_p + n_match; i++) { + // SLT_DBG(slot, "cache token %3zu: %6d '%s'\n", i, prompt_tokens[i], + // common_token_to_piece(ctx, prompt_tokens[i]).c_str()); + // } + + const int64_t kv_shift = (int64_t)head_p - (int64_t)head_c; + + llama_kv_cache_seq_rm(ctx, slot.id, head_p, head_c); + llama_kv_cache_seq_add(ctx, slot.id, head_c, head_c + n_match, kv_shift); + + for (size_t i = 0; i < n_match; i++) { + slot.cache_tokens[head_p + i] = slot.cache_tokens[head_c + i]; + slot.n_past++; + } + + head_c += n_match; + head_p += n_match; + } else { + head_c += 1; + } + } + + SLT_DBG(slot, "after context reuse, new slot.n_past = %d\n", slot.n_past); + } + } + } + + if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) { + // we have to evaluate at least 1 token to generate logits. + SLT_WRN(slot, + "need to evaluate at least 1 token to generate logits, n_past = %d, " + "n_prompt_tokens = %d\n", + slot.n_past, slot.n_prompt_tokens); + + slot.n_past--; + } + + slot.n_prompt_tokens_processed = 0; + } + + // non-causal tasks require to fit the entire prompt in the physical batch + if (slot.is_non_causal()) { + // cannot fit the prompt in the current batch - will try next iter + if (batch.n_tokens + slot.n_prompt_tokens > n_batch) { + continue; + } + } + + // keep only the common part + if (!llama_kv_cache_seq_rm(ctx, slot.id, slot.n_past, -1)) { + // could not partially delete (likely using a non-Transformer model) + llama_kv_cache_seq_rm(ctx, slot.id, -1, -1); + + // there is no common part left + slot.n_past = 0; + } + + SLT_INF(slot, "kv cache rm [%d, end)\n", slot.n_past); + + // remove the non-common part from the cache + slot.cache_tokens.resize(slot.n_past); + + // add prompt tokens for processing in the current batch + while (slot.n_past < slot.n_prompt_tokens && batch.n_tokens < n_batch) { + // without pooling, we want to output the embeddings for all the tokens in the batch + const bool need_embd = slot.task_type == SERVER_TASK_TYPE_EMBEDDING && + llama_pooling_type(slot.ctx) == LLAMA_POOLING_TYPE_NONE; + + common_batch_add(batch, prompt_tokens[slot.n_past], slot.n_past, {slot.id}, need_embd); + + if (slot.params.cache_prompt) { + slot.cache_tokens.push_back(prompt_tokens[slot.n_past]); + } + + slot.n_prompt_tokens_processed++; + slot.n_past++; + } + + SLT_INF(slot, "prompt processing progress, n_past = %d, n_tokens = %d, progress = %f\n", + slot.n_past, batch.n_tokens, (float)slot.n_prompt_tokens_processed / slot.n_prompt_tokens); + + // entire prompt has been processed + if (slot.n_past == slot.n_prompt_tokens) { + slot.state = SLOT_STATE_DONE_PROMPT; + + GGML_ASSERT(batch.n_tokens > 0); + + common_sampler_reset(slot.smpl); + + // Process all prompt tokens through sampler system + for (int i = 0; i < slot.n_prompt_tokens; ++i) { + common_sampler_accept(slot.smpl, prompt_tokens[i], false); + } + + // extract the logits only for the last token + batch.logits[batch.n_tokens - 1] = true; + + slot.n_decoded = 0; + slot.i_batch = batch.n_tokens - 1; + + SLT_INF(slot, "prompt done, n_past = %d, n_tokens = %d\n", slot.n_past, batch.n_tokens); + } + } + + if (batch.n_tokens >= n_batch) { + break; + } + } + } + + if (batch.n_tokens == 0) { + SRV_WRN("%s", "no tokens to decode\n"); + return; + } + + SRV_DBG("decoding batch, n_tokens = %d\n", batch.n_tokens); + + if (slot_batched) { + // make sure we're in the right embedding mode + llama_set_embeddings(ctx, slot_batched->is_non_causal()); + // apply lora, only need to do it once per batch + common_set_adapter_lora(ctx, slot_batched->lora); + } + + // process the created batch of tokens + for (int32_t i = 0; i < batch.n_tokens; i += n_batch) { + const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i); + + llama_batch batch_view = { + n_tokens, batch.token + i, nullptr, batch.pos + i, + batch.n_seq_id + i, batch.seq_id + i, batch.logits + i, + }; + + const int ret = llama_decode(ctx, batch_view); + metrics.on_decoded(slots); + + if (ret != 0) { + if (n_batch == 1 || ret < 0) { + // if you get here, it means the KV cache is full - try increasing it via the context size + SRV_ERR("failed to decode the batch: KV cache is full - try increasing it via the context size, i " + "= %d, n_batch = %d, ret = %d\n", + i, n_batch, ret); + for (auto &slot : slots) { + slot.release(); + send_error(slot, "Input prompt is too big compared to KV size. Please try increasing KV size."); + } + break; // break loop of n_batch + } + + // retry with half the batch size to try to find a free slot in the KV cache + n_batch /= 2; + i -= n_batch; + + SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size - try increasing " + "it via the context size or enable defragmentation, i = %d, n_batch = %d, ret = %d\n", + i, n_batch, ret); + + continue; // continue loop of n_batch + } + + for (auto &slot : slots) { + if (slot.i_batch < (int)i || slot.i_batch >= (int)(i + n_tokens)) { + continue; // continue loop of slots + } + + if (slot.state == SLOT_STATE_DONE_PROMPT) { + if (slot.task_type == SERVER_TASK_TYPE_EMBEDDING) { + // prompt evaluated for embedding + send_embedding(slot, batch_view); + slot.release(); + slot.i_batch = -1; + continue; // continue loop of slots + } + + if (slot.task_type == SERVER_TASK_TYPE_RERANK) { + send_rerank(slot, batch_view); + slot.release(); + slot.i_batch = -1; + continue; // continue loop of slots + } + + // prompt evaluated for next-token prediction + slot.state = SLOT_STATE_GENERATING; + } else if (slot.state != SLOT_STATE_GENERATING) { + continue; // continue loop of slots + } + + const int tok_idx = slot.i_batch - i; + + llama_token id = common_sampler_sample(slot.smpl, ctx, tok_idx); + + slot.i_batch = -1; + + common_sampler_accept(slot.smpl, id, true); + + slot.n_decoded += 1; + + const int64_t t_current = ggml_time_us(); + + if (slot.n_decoded == 1) { + slot.t_start_generation = t_current; + slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3; + metrics.on_prompt_eval(slot); + } + + slot.t_token_generation = (t_current - slot.t_start_generation) / 1e3; + + completion_token_output result; + result.tok = id; + result.text_to_send = common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok)); + result.prob = 1.0f; // TODO: set it here instead of doing inside populate_token_probs + + if (slot.params.sampling.n_probs > 0) { + populate_token_probs(slot, result, slot.params.post_sampling_probs, params_base.special, tok_idx); + } + + if (!process_token(result, slot)) { + // release slot because of stop condition + slot.release(); + slot.print_timings(); + send_final_response(slot); + metrics.on_prediction(slot); + continue; + } + } + + // do speculative decoding + for (auto &slot : slots) { + if (!slot.is_processing() || !slot.can_speculate()) { + continue; + } + + if (slot.state != SLOT_STATE_GENERATING) { + continue; + } + + // determine the max draft that fits the current slot state + int n_draft_max = slot.params.speculative.n_max; + + // note: n_past is not yet increased for the `id` token sampled above + // also, need to leave space for 1 extra token to allow context shifts + n_draft_max = std::min(n_draft_max, slot.n_ctx - slot.n_past - 2); + + if (slot.n_remaining > 0) { + n_draft_max = std::min(n_draft_max, slot.n_remaining - 1); + } + + SLT_DBG(slot, "max possible draft: %d\n", n_draft_max); + + if (n_draft_max < slot.params.speculative.n_min) { + SLT_DBG(slot, "the max possible draft is too small: %d < %d - skipping speculative decoding\n", + n_draft_max, slot.params.speculative.n_min); + + continue; + } + + llama_token id = slot.sampled; + + struct common_speculative_params params_spec; + params_spec.n_draft = n_draft_max; + params_spec.n_reuse = llama_n_ctx(slot.ctx_dft) - slot.params.speculative.n_max; + params_spec.p_min = slot.params.speculative.p_min; + + llama_tokens draft = common_speculative_gen_draft(slot.spec, params_spec, slot.cache_tokens, id); + + // ignore small drafts + if (slot.params.speculative.n_min > (int)draft.size()) { + SLT_DBG(slot, "ignoring small draft: %d < %d\n", (int)draft.size(), slot.params.speculative.n_min); + + continue; + } + + // construct the speculation batch + common_batch_clear(slot.batch_spec); + common_batch_add(slot.batch_spec, id, slot.n_past, {slot.id}, true); + + for (size_t i = 0; i < draft.size(); ++i) { + common_batch_add(slot.batch_spec, draft[i], slot.n_past + 1 + i, {slot.id}, true); + } + + SLT_DBG(slot, "decoding speculative batch, size = %d\n", slot.batch_spec.n_tokens); + + llama_decode(ctx, slot.batch_spec); + + // the accepted tokens from the speculation + const auto ids = common_sampler_sample_and_accept_n(slot.smpl, ctx, draft); + + slot.n_past += ids.size(); + slot.n_decoded += ids.size(); + + slot.cache_tokens.push_back(id); + slot.cache_tokens.insert(slot.cache_tokens.end(), ids.begin(), ids.end() - 1); + + llama_kv_cache_seq_rm(ctx, slot.id, slot.n_past, -1); + + for (size_t i = 0; i < ids.size(); ++i) { + completion_token_output result; + + result.tok = ids[i]; + result.text_to_send = + common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok)); + result.prob = 1.0f; // set later + + // TODO: set result.probs + + if (!process_token(result, slot)) { + // release slot because of stop condition + slot.release(); + slot.print_timings(); + send_final_response(slot); + metrics.on_prediction(slot); + break; + } + } + + SLT_DBG(slot, "accepted %d/%d draft tokens, new n_past = %d\n", (int)ids.size() - 1, (int)draft.size(), + slot.n_past); + } + } + + SRV_DBG("%s", "run slots completed\n"); + } + + json model_meta() const { + return json{ + {"vocab_type", llama_vocab_type(vocab)}, {"n_vocab", llama_vocab_n_tokens(vocab)}, + {"n_ctx_train", llama_model_n_ctx_train(model)}, {"n_embd", llama_model_n_embd(model)}, + {"n_params", llama_model_n_params(model)}, {"size", llama_model_size(model)}, + }; + } +}; diff --git a/src/main/cpp/utils.hpp b/src/main/cpp/utils.hpp new file mode 100644 index 000000000..603424b42 --- /dev/null +++ b/src/main/cpp/utils.hpp @@ -0,0 +1,856 @@ +#pragma once + +#include "base64.hpp" +#include "common.h" +#include "llama.h" +#include "log.h" + +#ifndef NDEBUG +// crash the server in debug mode, otherwise send an http 500 error +#define CPPHTTPLIB_NO_EXCEPTIONS 1 +#endif +// increase max payload length to allow use of larger context size +#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576 +// #include "httplib.h" + +// Change JSON_ASSERT from assert() to GGML_ASSERT: +#define JSON_ASSERT GGML_ASSERT +#include "nlohmann/json.hpp" + +#include "chat.h" + +#include +#include +#include +#include +#include + +#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo" + +using json = nlohmann::ordered_json; + +#define SLT_INF(slot, fmt, ...) \ + LOG_INF("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__) +#define SLT_WRN(slot, fmt, ...) \ + LOG_WRN("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__) +#define SLT_ERR(slot, fmt, ...) \ + LOG_ERR("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__) +#define SLT_DBG(slot, fmt, ...) \ + LOG_DBG("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__) + +#define SRV_INF(fmt, ...) LOG_INF("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SRV_WRN(fmt, ...) LOG_WRN("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SRV_ERR(fmt, ...) LOG_ERR("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SRV_DBG(fmt, ...) LOG_DBG("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__) + +#define QUE_INF(fmt, ...) LOG_INF("que %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define QUE_WRN(fmt, ...) LOG_WRN("que %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define QUE_ERR(fmt, ...) LOG_ERR("que %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define QUE_DBG(fmt, ...) LOG_DBG("que %12.*s: " fmt, 12, __func__, __VA_ARGS__) + +template static T json_value(const json &body, const std::string &key, const T &default_value) { + // Fallback null to default value + if (body.contains(key) && !body.at(key).is_null()) { + try { + return body.at(key); + } catch (NLOHMANN_JSON_NAMESPACE::detail::type_error const &) { + LOG_WRN("Wrong type supplied for parameter '%s'. Expected '%s', using default value\n", key.c_str(), + json(default_value).type_name()); + return default_value; + } + } else { + return default_value; + } +} + +const static std::string build_info("b" + std::to_string(LLAMA_BUILD_NUMBER) + "-" + LLAMA_COMMIT); + +// +// tokenizer and input processing utils +// + +static bool json_is_array_of_numbers(const json &data) { + if (data.is_array()) { + for (const auto &e : data) { + if (!e.is_number_integer()) { + return false; + } + } + return true; + } + return false; +} + +// is array having BOTH numbers & strings? +static bool json_is_array_of_mixed_numbers_strings(const json &data) { + bool seen_string = false; + bool seen_number = false; + if (data.is_array()) { + for (const auto &e : data) { + seen_string |= e.is_string(); + seen_number |= e.is_number_integer(); + if (seen_number && seen_string) { + return true; + } + } + } + return false; +} + +// get value by path(key1 / key2) +static json json_get_nested_values(const std::vector &paths, const json &js) { + json result = json::object(); + + for (const std::string &path : paths) { + json current = js; + const auto keys = string_split(path, /*separator*/ '/'); + bool valid_path = true; + for (const std::string &k : keys) { + if (valid_path && current.is_object() && current.contains(k)) { + current = current[k]; + } else { + valid_path = false; + } + } + if (valid_path) { + result[path] = current; + } + } + return result; +} + +/** + * this handles 2 cases: + * - only string, example: "string" + * - mixed string and tokens, example: [12, 34, "string", 56, 78] + */ +static llama_tokens tokenize_mixed(const llama_vocab *vocab, const json &json_prompt, bool add_special, + bool parse_special) { + // If `add_bos` is true, we only add BOS, when json_prompt is a string, + // or the first element of the json_prompt array is a string. + llama_tokens prompt_tokens; + + if (json_prompt.is_array()) { + bool first = true; + for (const auto &p : json_prompt) { + if (p.is_string()) { + auto s = p.template get(); + + llama_tokens p; + if (first) { + p = common_tokenize(vocab, s, add_special, parse_special); + first = false; + } else { + p = common_tokenize(vocab, s, false, parse_special); + } + + prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end()); + } else { + if (first) { + first = false; + } + + prompt_tokens.push_back(p.template get()); + } + } + } else { + auto s = json_prompt.template get(); + prompt_tokens = common_tokenize(vocab, s, add_special, parse_special); + } + + return prompt_tokens; +} + +/** + * break the input "prompt" object into multiple prompt if needed, then tokenize them + * this supports these cases: + * - "prompt": "string" + * - "prompt": [12, 34, 56] + * - "prompt": [12, 34, "string", 56, 78] + * and multiple prompts (multi-tasks): + * - "prompt": ["string1", "string2"] + * - "prompt": ["string1", [12, 34, 56]] + * - "prompt": [[12, 34, 56], [78, 90, 12]] + * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56]] + */ +static std::vector tokenize_input_prompts(const llama_vocab *vocab, const json &json_prompt, + bool add_special, bool parse_special) { + std::vector result; + if (json_prompt.is_string() || json_is_array_of_mixed_numbers_strings(json_prompt)) { + // string or mixed + result.push_back(tokenize_mixed(vocab, json_prompt, add_special, parse_special)); + } else if (json_is_array_of_numbers(json_prompt)) { + // array of tokens + result.push_back(json_prompt.get()); + } else if (json_prompt.is_array()) { + // array of prompts + result.reserve(json_prompt.size()); + for (const auto &p : json_prompt) { + if (p.is_string() || json_is_array_of_mixed_numbers_strings(p)) { + result.push_back(tokenize_mixed(vocab, p, add_special, parse_special)); + } else if (json_is_array_of_numbers(p)) { + // array of tokens + result.push_back(p.get()); + } else { + throw std::runtime_error( + "element of \"prompt\" must be a string, an list of tokens, or a list of mixed strings & tokens"); + } + } + } else { + throw std::runtime_error( + "\"prompt\" must be a string, an list of tokens, a list of mixed strings & tokens, or a list of prompts"); + } + if (result.empty()) { + throw std::runtime_error("\"prompt\" must not be empty"); + } + return result; +} + +// return the last index of character that can form a valid string +// if the last character is potentially cut in half, return the index before the cut +// if validate_utf8(text) == text.size(), then the whole text is valid utf8 +static size_t validate_utf8(const std::string &text) { + size_t len = text.size(); + if (len == 0) + return 0; + + // Check the last few bytes to see if a multi-byte character is cut off + for (size_t i = 1; i <= 4 && i <= len; ++i) { + unsigned char c = text[len - i]; + // Check for start of a multi-byte sequence from the end + if ((c & 0xE0) == 0xC0) { + // 2-byte character start: 110xxxxx + // Needs at least 2 bytes + if (i < 2) + return len - i; + } else if ((c & 0xF0) == 0xE0) { + // 3-byte character start: 1110xxxx + // Needs at least 3 bytes + if (i < 3) + return len - i; + } else if ((c & 0xF8) == 0xF0) { + // 4-byte character start: 11110xxx + // Needs at least 4 bytes + if (i < 4) + return len - i; + } + } + + // If no cut-off multi-byte character is found, return full length + return len; +} + +// +// template utils +// + +// format rerank task: [BOS]query[EOS][SEP]doc[EOS] +static llama_tokens format_rerank(const struct llama_vocab *vocab, const llama_tokens &query, const llama_tokens &doc) { + llama_tokens result; + + result.reserve(doc.size() + query.size() + 4); + result.push_back(llama_vocab_bos(vocab)); + result.insert(result.end(), query.begin(), query.end()); + result.push_back(llama_vocab_eos(vocab)); + result.push_back(llama_vocab_sep(vocab)); + result.insert(result.end(), doc.begin(), doc.end()); + result.push_back(llama_vocab_eos(vocab)); + + return result; +} + +// format infill task +static llama_tokens format_infill(const llama_vocab *vocab, const json &input_prefix, const json &input_suffix, + const json &input_extra, const int n_batch, const int n_predict, const int n_ctx, + const bool spm_infill, const llama_tokens &tokens_prompt) { + // TODO: optimize this block by reducing memory allocations and movement + + // use FIM repo-level pattern: + // ref: https://arxiv.org/pdf/2409.12186 + // + // [FIM_REP]myproject + // [FIM_SEP]filename0 + // extra chunk 0 + // [FIM_SEP]filename1 + // extra chunk 1 + // ... + // [FIM_SEP]filename + // [FIM_PRE]prefix[FIM_SUF]suffix[FIM_MID]prompt + // + llama_tokens extra_tokens; + extra_tokens.reserve(n_ctx); + + auto tokens_prefix = tokenize_mixed(vocab, input_prefix, false, false); + auto tokens_suffix = tokenize_mixed(vocab, input_suffix, false, false); + + if (llama_vocab_fim_rep(vocab) != LLAMA_TOKEN_NULL) { + // TODO: make project name an input + static const auto k_fim_repo = common_tokenize(vocab, "myproject\n", false, false); + + extra_tokens.push_back(llama_vocab_fim_rep(vocab)); + extra_tokens.insert(extra_tokens.end(), k_fim_repo.begin(), k_fim_repo.end()); + } + for (const auto &chunk : input_extra) { + // { "text": string, "filename": string } + const std::string text = json_value(chunk, "text", std::string()); + const std::string filename = json_value(chunk, "filename", std::string("tmp")); + + if (llama_vocab_fim_sep(vocab) != LLAMA_TOKEN_NULL) { + const auto k_fim_file = common_tokenize(vocab, filename + "\n", false, false); + + extra_tokens.insert(extra_tokens.end(), llama_vocab_fim_sep(vocab)); + extra_tokens.insert(extra_tokens.end(), k_fim_file.begin(), k_fim_file.end()); + } else { + // chunk separator in binary form to avoid confusing the AI + static const char k_chunk_prefix_str[] = {0x0a, 0x0a, 0x2d, 0x2d, 0x2d, 0x20, 0x73, 0x6e, 0x69, 0x70, + 0x70, 0x65, 0x74, 0x20, 0x2d, 0x2d, 0x2d, 0x0a, 0x0a, 0x00}; + static const auto k_chunk_prefix_tokens = common_tokenize(vocab, k_chunk_prefix_str, false, false); + + extra_tokens.insert(extra_tokens.end(), k_chunk_prefix_tokens.begin(), k_chunk_prefix_tokens.end()); + } + + const auto chunk_tokens = common_tokenize(vocab, text, false, false); + extra_tokens.insert(extra_tokens.end(), chunk_tokens.begin(), chunk_tokens.end()); + } + + if (llama_vocab_fim_sep(vocab) != LLAMA_TOKEN_NULL) { + // TODO: current filename + static const auto k_fim_file = common_tokenize(vocab, "filename\n", false, false); + + extra_tokens.insert(extra_tokens.end(), llama_vocab_fim_sep(vocab)); + extra_tokens.insert(extra_tokens.end(), k_fim_file.begin(), k_fim_file.end()); + } + + // for now pick FIM context to fit in a batch (ratio prefix:suffix = 3:1, TODO: configurable?) + const int n_prefix_take = std::min(tokens_prefix.size(), 3 * (n_batch / 4)); + const int n_suffix_take = + std::min(tokens_suffix.size(), std::max(0, (n_batch / 4) - (2 + tokens_prompt.size()))); + + SRV_DBG("n_prefix_take = %d, n_suffix_take = %d, total = %d\n", n_prefix_take, n_suffix_take, + (n_prefix_take + n_suffix_take)); + + // fill the rest of the context with extra chunks + const int n_extra_take = std::min(std::max(0, n_ctx - (n_batch)-2 * n_predict), extra_tokens.size()); + + tokens_prefix.erase(tokens_prefix.begin(), tokens_prefix.begin() + tokens_prefix.size() - n_prefix_take); + tokens_suffix.resize(n_suffix_take); + + tokens_prefix.insert(tokens_prefix.begin(), llama_vocab_fim_pre(vocab)); + tokens_prefix.insert(tokens_prefix.end(), tokens_prompt.begin(), tokens_prompt.end()); + tokens_suffix.insert(tokens_suffix.begin(), llama_vocab_fim_suf(vocab)); + + auto embd_inp = spm_infill ? tokens_suffix : tokens_prefix; + auto embd_end = spm_infill ? tokens_prefix : tokens_suffix; + + if (llama_vocab_get_add_bos(vocab)) { + embd_inp.insert(embd_inp.begin(), llama_vocab_bos(vocab)); + } + + SRV_DBG("extra: n_ctx = %d, n_extra_take = %d, n_extra = %d\n", n_ctx, n_extra_take, (int)extra_tokens.size()); + + // put the extra context before the FIM prefix + embd_inp.insert(embd_inp.begin(), extra_tokens.end() - n_extra_take, extra_tokens.end()); + + embd_inp.insert(embd_inp.end(), embd_end.begin(), embd_end.end()); + embd_inp.push_back(llama_vocab_fim_mid(vocab)); + + return embd_inp; +} + +// +// base64 utils (TODO: move to common in the future) +// + +static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +static inline bool is_base64(uint8_t c) { return (isalnum(c) || (c == '+') || (c == '/')); } + +static inline std::vector base64_decode(const std::string &encoded_string) { + int i = 0; + int j = 0; + int in_ = 0; + + int in_len = encoded_string.size(); + + uint8_t char_array_4[4]; + uint8_t char_array_3[3]; + + std::vector ret; + + while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { + char_array_4[i++] = encoded_string[in_]; + in_++; + if (i == 4) { + for (i = 0; i < 4; i++) { + char_array_4[i] = base64_chars.find(char_array_4[i]); + } + + char_array_3[0] = ((char_array_4[0]) << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (i = 0; (i < 3); i++) { + ret.push_back(char_array_3[i]); + } + + i = 0; + } + } + + if (i) { + for (j = i; j < 4; j++) { + char_array_4[j] = 0; + } + + for (j = 0; j < 4; j++) { + char_array_4[j] = base64_chars.find(char_array_4[j]); + } + + char_array_3[0] = ((char_array_4[0]) << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (j = 0; j < i - 1; j++) { + ret.push_back(char_array_3[j]); + } + } + + return ret; +} + +// +// random string / id +// + +static std::string random_string() { + static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + + std::random_device rd; + std::mt19937 generator(rd()); + + std::string result(32, ' '); + + for (int i = 0; i < 32; ++i) { + result[i] = str[generator() % str.size()]; + } + + return result; +} + +static std::string gen_chatcmplid() { return "chatcmpl-" + random_string(); } + +// +// other common utils +// + +static bool ends_with(const std::string &str, const std::string &suffix) { + return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); +} + +static size_t find_partial_stop_string(const std::string &stop, const std::string &text) { + if (!text.empty() && !stop.empty()) { + const char text_last_char = text.back(); + for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) { + if (stop[char_index] == text_last_char) { + const std::string current_partial = stop.substr(0, char_index + 1); + if (ends_with(text, current_partial)) { + return text.size() - char_index - 1; + } + } + } + } + + return std::string::npos; +} + +// TODO: reuse llama_detokenize +template static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end) { + std::string ret; + for (; begin != end; ++begin) { + ret += common_token_to_piece(ctx, *begin); + } + + return ret; +} + +// format incomplete utf-8 multibyte character for output +static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token) { + std::string out = token == LLAMA_TOKEN_NULL ? "" : common_token_to_piece(ctx, token); + + // if the size is 1 and first bit is 1, meaning it's a partial character + // (size > 1 meaning it's already a known token) + if (out.size() == 1 && (out[0] & 0x80) == 0x80) { + std::stringstream ss; + ss << std::hex << (out[0] & 0xff); + std::string res(ss.str()); + out = "byte: \\x" + res; + } + + return out; +} + +// static bool server_sent_event(httplib::DataSink & sink, const char * event, const json & data) { +// const std::string str = +// std::string(event) + ": " + +// data.dump(-1, ' ', false, json::error_handler_t::replace) + +// "\n\n"; // required by RFC 8895 - A message is terminated by a blank line (two line terminators in a row). +// +// LOG_DBG("data stream, to_send: %s", str.c_str()); +// +// return sink.write(str.c_str(), str.size()); +// } + +// +// OAI utils +// + +static json oaicompat_completion_params_parse(const json &body) { + json llama_params; + + if (!body.contains("prompt")) { + throw std::runtime_error("\"prompt\" is required"); + } + + // Handle "stop" field + if (body.contains("stop") && body.at("stop").is_string()) { + llama_params["stop"] = json::array({body.at("stop").get()}); + } else { + llama_params["stop"] = json_value(body, "stop", json::array()); + } + + // Handle "n" field + int n_choices = json_value(body, "n", 1); + if (n_choices != 1) { + throw std::runtime_error("Only one completion choice is allowed"); + } + + // Handle "echo" field + if (json_value(body, "echo", false)) { + throw std::runtime_error("Only no echo is supported"); + } + + // Params supported by OAI but unsupported by llama.cpp + static const std::vector unsupported_params{"best_of", "suffix"}; + for (const auto ¶m : unsupported_params) { + if (body.contains(param)) { + throw std::runtime_error("Unsupported param: " + param); + } + } + + // Copy remaining properties to llama_params + for (const auto &item : body.items()) { + // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens" + if (!llama_params.contains(item.key()) || item.key() == "n_predict") { + llama_params[item.key()] = item.value(); + } + } + + return llama_params; +} + +static json oaicompat_completion_params_parse(const json &body, /* openai api json semantics */ + bool use_jinja, common_reasoning_format reasoning_format, + const struct common_chat_templates *tmpls) { + json llama_params; + + auto tools = json_value(body, "tools", json()); + auto stream = json_value(body, "stream", false); + + if (tools.is_array() && !tools.empty()) { + if (stream) { + throw std::runtime_error("Cannot use tools with stream"); + } + if (!use_jinja) { + throw std::runtime_error("tools param requires --jinja flag"); + } + } + if (!use_jinja) { + if (body.contains("tool_choice") && !body.at("tool_choice").is_null()) { + throw std::runtime_error("Unsupported param: tool_choice"); + } + } + + // Handle "stop" field + if (body.contains("stop") && body.at("stop").is_string()) { + llama_params["stop"] = json::array({body.at("stop").get()}); + } else { + llama_params["stop"] = json_value(body, "stop", json::array()); + } + + auto json_schema = json_value(body, "json_schema", json()); + auto grammar = json_value(body, "grammar", std::string()); + if (!json_schema.is_null() && !grammar.empty()) { + throw std::runtime_error("Cannot use both json_schema and grammar"); + } + + // Handle "response_format" field + if (body.contains("response_format")) { + json response_format = json_value(body, "response_format", json::object()); + std::string response_type = json_value(response_format, "type", std::string()); + if (response_type == "json_object") { + json_schema = json_value(response_format, "schema", json::object()); + } else if (response_type == "json_schema") { + auto schema_wrapper = json_value(response_format, "json_schema", json::object()); + json_schema = json_value(schema_wrapper, "schema", json::object()); + } else if (!response_type.empty() && response_type != "text") { + throw std::runtime_error("response_format type must be one of \"text\" or \"json_object\", but got: " + + response_type); + } + } + + common_chat_templates_inputs inputs; + inputs.messages = common_chat_msgs_parse_oaicompat(body.at("messages")); + inputs.tools = common_chat_tools_parse_oaicompat(tools); + inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(json_value(body, "tool_choice", std::string("auto"))); + inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump(); + inputs.grammar = grammar; + inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true); + inputs.use_jinja = use_jinja; + inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", false); + inputs.extract_reasoning = reasoning_format != COMMON_REASONING_FORMAT_NONE; + inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true); + if (!inputs.tools.empty() && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && body.contains("grammar")) { + throw std::runtime_error("Cannot use custom grammar constraints with tools."); + } + + // Apply chat template to the list of messages + auto chat_params = common_chat_templates_apply(tmpls, inputs); + + llama_params["chat_format"] = static_cast(chat_params.format); + llama_params["prompt"] = chat_params.prompt; + llama_params["grammar"] = chat_params.grammar; + llama_params["grammar_lazy"] = chat_params.grammar_lazy; + auto grammar_triggers = json::array(); + for (const auto &trigger : chat_params.grammar_triggers) { + grammar_triggers.push_back(trigger.to_json()); + } + llama_params["grammar_triggers"] = grammar_triggers; + llama_params["preserved_tokens"] = chat_params.preserved_tokens; + for (const auto &stop : chat_params.additional_stops) { + llama_params["stop"].push_back(stop); + } + + // Handle "n" field + int n_choices = json_value(body, "n", 1); + if (n_choices != 1) { + throw std::runtime_error("Only one completion choice is allowed"); + } + + // Handle "logprobs" field + // TODO: The response format of this option is not yet OAI-compatible, but seems like no one really using it; We may + // need to fix it in the future + if (json_value(body, "logprobs", false)) { + llama_params["n_probs"] = json_value(body, "top_logprobs", 20); + } else if (body.contains("top_logprobs") && !body.at("top_logprobs").is_null()) { + throw std::runtime_error("top_logprobs requires logprobs to be set to true"); + } + + // Copy remaining properties to llama_params + // This allows user to use llama.cpp-specific params like "mirostat", ... via OAI endpoint. + // See "launch_slot_with_task()" for a complete list of params supported by llama.cpp + for (const auto &item : body.items()) { + // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens" + if (!llama_params.contains(item.key()) || item.key() == "n_predict") { + llama_params[item.key()] = item.value(); + } + } + + return llama_params; +} + +static json format_embeddings_response_oaicompat(const json &request, const json &embeddings, bool use_base64 = false) { + json data = json::array(); + int32_t n_tokens = 0; + int i = 0; + for (const auto &elem : embeddings) { + json embedding_obj; + + if (use_base64) { + const auto &vec = json_value(elem, "embedding", json::array()).get>(); + const char *data_ptr = reinterpret_cast(vec.data()); + size_t data_size = vec.size() * sizeof(float); + embedding_obj = {{"embedding", base64::encode(data_ptr, data_size)}, + {"index", i++}, + {"object", "embedding"}, + {"encoding_format", "base64"}}; + } else { + embedding_obj = { + {"embedding", json_value(elem, "embedding", json::array())}, {"index", i++}, {"object", "embedding"}}; + } + data.push_back(embedding_obj); + + n_tokens += json_value(elem, "tokens_evaluated", 0); + } + + json res = json{{"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, + {"object", "list"}, + {"usage", json{{"prompt_tokens", n_tokens}, {"total_tokens", n_tokens}}}, + {"data", data}}; + + return res; +} + +static json format_response_rerank(const json &request, const json &ranks, bool is_tei_format, + std::vector &texts) { + json res; + if (is_tei_format) { + // TEI response format + res = json::array(); + bool return_text = json_value(request, "return_text", false); + for (const auto &rank : ranks) { + int index = json_value(rank, "index", 0); + json elem = json{ + {"index", index}, + {"score", json_value(rank, "score", 0.0)}, + }; + if (return_text) { + elem["text"] = std::move(texts[index]); + } + res.push_back(elem); + } + } else { + // Jina response format + json results = json::array(); + int32_t n_tokens = 0; + for (const auto &rank : ranks) { + results.push_back(json{ + {"index", json_value(rank, "index", 0)}, + {"relevance_score", json_value(rank, "score", 0.0)}, + }); + + n_tokens += json_value(rank, "tokens_evaluated", 0); + } + + res = json{{"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))}, + {"object", "list"}, + {"usage", json{{"prompt_tokens", n_tokens}, {"total_tokens", n_tokens}}}, + {"results", results}}; + } + + return res; +} + +static bool is_valid_utf8(const std::string &str) { + const unsigned char *bytes = reinterpret_cast(str.data()); + const unsigned char *end = bytes + str.length(); + + while (bytes < end) { + if (*bytes <= 0x7F) { + // 1-byte sequence (0xxxxxxx) + bytes++; + } else if ((*bytes & 0xE0) == 0xC0) { + // 2-byte sequence (110xxxxx 10xxxxxx) + if (end - bytes < 2 || (bytes[1] & 0xC0) != 0x80) + return false; + bytes += 2; + } else if ((*bytes & 0xF0) == 0xE0) { + // 3-byte sequence (1110xxxx 10xxxxxx 10xxxxxx) + if (end - bytes < 3 || (bytes[1] & 0xC0) != 0x80 || (bytes[2] & 0xC0) != 0x80) + return false; + bytes += 3; + } else if ((*bytes & 0xF8) == 0xF0) { + // 4-byte sequence (11110xxx 10xxxxxx 10xxxxxx 10xxxxxx) + if (end - bytes < 4 || (bytes[1] & 0xC0) != 0x80 || (bytes[2] & 0xC0) != 0x80 || (bytes[3] & 0xC0) != 0x80) + return false; + bytes += 4; + } else { + // Invalid UTF-8 lead byte + return false; + } + } + + return true; +} + +static json format_tokenizer_response(const json &tokens) { return json{{"tokens", tokens}}; } + +static json format_detokenized_response(const std::string &content) { return json{{"content", content}}; } + +static json format_logit_bias(const std::vector &logit_bias) { + json data = json::array(); + for (const auto &lb : logit_bias) { + data.push_back(json{ + {"bias", lb.bias}, + {"token", lb.token}, + }); + } + return data; +} + +static std::string safe_json_to_str(const json &data) { + return data.dump(-1, ' ', false, json::error_handler_t::replace); +} + +static std::vector get_token_probabilities(llama_context *ctx, int idx) { + std::vector cur; + const auto *logits = llama_get_logits_ith(ctx, idx); + + const llama_model *model = llama_get_model(ctx); + const llama_vocab *vocab = llama_model_get_vocab(model); + + const int n_vocab = llama_vocab_n_tokens(vocab); + + cur.resize(n_vocab); + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + cur[token_id] = llama_token_data{token_id, logits[token_id], 0.0f}; + } + + // sort tokens by logits + std::sort(cur.begin(), cur.end(), + [](const llama_token_data &a, const llama_token_data &b) { return a.logit > b.logit; }); + + // apply softmax + float max_l = cur[0].logit; + float cum_sum = 0.0f; + for (size_t i = 0; i < cur.size(); ++i) { + float p = expf(cur[i].logit - max_l); + cur[i].p = p; + cum_sum += p; + } + for (size_t i = 0; i < cur.size(); ++i) { + cur[i].p /= cum_sum; + } + + return cur; +} + +static bool are_lora_equal(const std::vector &l1, + const std::vector &l2) { + if (l1.size() != l2.size()) { + return false; + } + for (size_t i = 0; i < l1.size(); ++i) { + // we don't check lora.path to reduce the time complexity + if (l1[i].scale != l2[i].scale || l1[i].ptr != l2[i].ptr) { + return false; + } + } + return true; +} + +// parse lora config from JSON request, returned a copy of lora_base with updated scale +static std::vector parse_lora_request(const std::vector &lora_base, + const json &data) { + std::vector lora(lora_base); + int max_idx = lora.size(); + + // clear existing value + for (auto &entry : lora) { + entry.scale = 0.0f; + } + + // set value + for (const auto &entry : data) { + int id = json_value(entry, "id", -1); + float scale = json_value(entry, "scale", 0.0f); + if (0 <= id && id < max_idx) { + lora[id].scale = scale; + } else { + throw std::runtime_error("invalid adapter id"); + } + } + + return lora; +} \ No newline at end of file diff --git a/src/main/java/de/kherud/llama/CliParameters.java b/src/main/java/de/kherud/llama/CliParameters.java new file mode 100644 index 000000000..4142628e2 --- /dev/null +++ b/src/main/java/de/kherud/llama/CliParameters.java @@ -0,0 +1,40 @@ +package de.kherud.llama; + +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +abstract class CliParameters { + + final Map parameters = new HashMap<>(); + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String key : parameters.keySet()) { + String value = parameters.get(key); + builder.append(key).append(" "); + if (value != null) { + builder.append(value).append(" "); + } + } + return builder.toString(); + } + + public String[] toArray() { + List result = new ArrayList<>(); + result.add(""); // c args contain the program name as the first argument, so we add an empty entry + for (String key : parameters.keySet()) { + result.add(key); + String value = parameters.get(key); + if (value != null) { + result.add(value); + } + } + return result.toArray(new String[0]); + } + +} diff --git a/src/main/java/de/kherud/llama/InferenceParameters.java b/src/main/java/de/kherud/llama/InferenceParameters.java new file mode 100644 index 000000000..41f74cc9e --- /dev/null +++ b/src/main/java/de/kherud/llama/InferenceParameters.java @@ -0,0 +1,546 @@ +package de.kherud.llama; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import de.kherud.llama.args.MiroStat; +import de.kherud.llama.args.Sampler; + +/** + * Parameters used throughout inference of a {@link LlamaModel}, e.g., {@link LlamaModel#generate(InferenceParameters)} + * and + * {@link LlamaModel#complete(InferenceParameters)}. + */ +@SuppressWarnings("unused") +public final class InferenceParameters extends JsonParameters { + + private static final String PARAM_PROMPT = "prompt"; + private static final String PARAM_INPUT_PREFIX = "input_prefix"; + private static final String PARAM_INPUT_SUFFIX = "input_suffix"; + private static final String PARAM_CACHE_PROMPT = "cache_prompt"; + private static final String PARAM_N_PREDICT = "n_predict"; + private static final String PARAM_TOP_K = "top_k"; + private static final String PARAM_TOP_P = "top_p"; + private static final String PARAM_MIN_P = "min_p"; + private static final String PARAM_TFS_Z = "tfs_z"; + private static final String PARAM_TYPICAL_P = "typical_p"; + private static final String PARAM_TEMPERATURE = "temperature"; + private static final String PARAM_DYNATEMP_RANGE = "dynatemp_range"; + private static final String PARAM_DYNATEMP_EXPONENT = "dynatemp_exponent"; + private static final String PARAM_REPEAT_LAST_N = "repeat_last_n"; + private static final String PARAM_REPEAT_PENALTY = "repeat_penalty"; + private static final String PARAM_FREQUENCY_PENALTY = "frequency_penalty"; + private static final String PARAM_PRESENCE_PENALTY = "presence_penalty"; + private static final String PARAM_MIROSTAT = "mirostat"; + private static final String PARAM_MIROSTAT_TAU = "mirostat_tau"; + private static final String PARAM_MIROSTAT_ETA = "mirostat_eta"; + private static final String PARAM_PENALIZE_NL = "penalize_nl"; + private static final String PARAM_N_KEEP = "n_keep"; + private static final String PARAM_SEED = "seed"; + private static final String PARAM_N_PROBS = "n_probs"; + private static final String PARAM_MIN_KEEP = "min_keep"; + private static final String PARAM_GRAMMAR = "grammar"; + private static final String PARAM_PENALTY_PROMPT = "penalty_prompt"; + private static final String PARAM_IGNORE_EOS = "ignore_eos"; + private static final String PARAM_LOGIT_BIAS = "logit_bias"; + private static final String PARAM_STOP = "stop"; + private static final String PARAM_SAMPLERS = "samplers"; + private static final String PARAM_STREAM = "stream"; + private static final String PARAM_USE_CHAT_TEMPLATE = "use_chat_template"; + private static final String PARAM_USE_JINJA = "use_jinja"; + private static final String PARAM_MESSAGES = "messages"; + + public InferenceParameters(String prompt) { + // we always need a prompt + setPrompt(prompt); + } + + /** + * Set the prompt to start generation with (default: empty) + */ + public InferenceParameters setPrompt(String prompt) { + parameters.put(PARAM_PROMPT, toJsonString(prompt)); + return this; + } + + /** + * Set a prefix for infilling (default: empty) + */ + public InferenceParameters setInputPrefix(String inputPrefix) { + parameters.put(PARAM_INPUT_PREFIX, toJsonString(inputPrefix)); + return this; + } + + /** + * Set a suffix for infilling (default: empty) + */ + public InferenceParameters setInputSuffix(String inputSuffix) { + parameters.put(PARAM_INPUT_SUFFIX, toJsonString(inputSuffix)); + return this; + } + + /** + * Whether to remember the prompt to avoid reprocessing it + */ + public InferenceParameters setCachePrompt(boolean cachePrompt) { + parameters.put(PARAM_CACHE_PROMPT, String.valueOf(cachePrompt)); + return this; + } + + /** + * Set the number of tokens to predict (default: -1, -1 = infinity, -2 = until context filled) + */ + public InferenceParameters setNPredict(int nPredict) { + parameters.put(PARAM_N_PREDICT, String.valueOf(nPredict)); + return this; + } + + /** + * Set top-k sampling (default: 40, 0 = disabled) + */ + public InferenceParameters setTopK(int topK) { + parameters.put(PARAM_TOP_K, String.valueOf(topK)); + return this; + } + + /** + * Set top-p sampling (default: 0.9, 1.0 = disabled) + */ + public InferenceParameters setTopP(float topP) { + parameters.put(PARAM_TOP_P, String.valueOf(topP)); + return this; + } + + /** + * Set min-p sampling (default: 0.1, 0.0 = disabled) + */ + public InferenceParameters setMinP(float minP) { + parameters.put(PARAM_MIN_P, String.valueOf(minP)); + return this; + } + + /** + * Set tail free sampling, parameter z (default: 1.0, 1.0 = disabled) + */ + public InferenceParameters setTfsZ(float tfsZ) { + parameters.put(PARAM_TFS_Z, String.valueOf(tfsZ)); + return this; + } + + /** + * Set locally typical sampling, parameter p (default: 1.0, 1.0 = disabled) + */ + public InferenceParameters setTypicalP(float typicalP) { + parameters.put(PARAM_TYPICAL_P, String.valueOf(typicalP)); + return this; + } + + /** + * Set the temperature (default: 0.8) + */ + public InferenceParameters setTemperature(float temperature) { + parameters.put(PARAM_TEMPERATURE, String.valueOf(temperature)); + return this; + } + + /** + * Set the dynamic temperature range (default: 0.0, 0.0 = disabled) + */ + public InferenceParameters setDynamicTemperatureRange(float dynatempRange) { + parameters.put(PARAM_DYNATEMP_RANGE, String.valueOf(dynatempRange)); + return this; + } + + /** + * Set the dynamic temperature exponent (default: 1.0) + */ + public InferenceParameters setDynamicTemperatureExponent(float dynatempExponent) { + parameters.put(PARAM_DYNATEMP_EXPONENT, String.valueOf(dynatempExponent)); + return this; + } + + /** + * Set the last n tokens to consider for penalties (default: 64, 0 = disabled, -1 = ctx_size) + */ + public InferenceParameters setRepeatLastN(int repeatLastN) { + parameters.put(PARAM_REPEAT_LAST_N, String.valueOf(repeatLastN)); + return this; + } + + /** + * Set the penalty of repeated sequences of tokens (default: 1.0, 1.0 = disabled) + */ + public InferenceParameters setRepeatPenalty(float repeatPenalty) { + parameters.put(PARAM_REPEAT_PENALTY, String.valueOf(repeatPenalty)); + return this; + } + + /** + * Set the repetition alpha frequency penalty (default: 0.0, 0.0 = disabled) + */ + public InferenceParameters setFrequencyPenalty(float frequencyPenalty) { + parameters.put(PARAM_FREQUENCY_PENALTY, String.valueOf(frequencyPenalty)); + return this; + } + + /** + * Set the repetition alpha presence penalty (default: 0.0, 0.0 = disabled) + */ + public InferenceParameters setPresencePenalty(float presencePenalty) { + parameters.put(PARAM_PRESENCE_PENALTY, String.valueOf(presencePenalty)); + return this; + } + + /** + * Set MiroStat sampling strategies. + */ + public InferenceParameters setMiroStat(MiroStat mirostat) { + parameters.put(PARAM_MIROSTAT, String.valueOf(mirostat.ordinal())); + return this; + } + + /** + * Set the MiroStat target entropy, parameter tau (default: 5.0) + */ + public InferenceParameters setMiroStatTau(float mirostatTau) { + parameters.put(PARAM_MIROSTAT_TAU, String.valueOf(mirostatTau)); + return this; + } + + /** + * Set the MiroStat learning rate, parameter eta (default: 0.1) + */ + public InferenceParameters setMiroStatEta(float mirostatEta) { + parameters.put(PARAM_MIROSTAT_ETA, String.valueOf(mirostatEta)); + return this; + } + + /** + * Whether to penalize newline tokens + */ + public InferenceParameters setPenalizeNl(boolean penalizeNl) { + parameters.put(PARAM_PENALIZE_NL, String.valueOf(penalizeNl)); + return this; + } + + /** + * Set the number of tokens to keep from the initial prompt (default: 0, -1 = all) + */ + public InferenceParameters setNKeep(int nKeep) { + parameters.put(PARAM_N_KEEP, String.valueOf(nKeep)); + return this; + } + + /** + * Set the RNG seed (default: -1, use random seed for < 0) + */ + public InferenceParameters setSeed(int seed) { + parameters.put(PARAM_SEED, String.valueOf(seed)); + return this; + } + + /** + * Set the amount top tokens probabilities to output if greater than 0. + */ + public InferenceParameters setNProbs(int nProbs) { + parameters.put(PARAM_N_PROBS, String.valueOf(nProbs)); + return this; + } + + /** + * Set the amount of tokens the samplers should return at least (0 = disabled) + */ + public InferenceParameters setMinKeep(int minKeep) { + parameters.put(PARAM_MIN_KEEP, String.valueOf(minKeep)); + return this; + } + + /** + * Set BNF-like grammar to constrain generations (see samples in grammars/ dir) + */ + public InferenceParameters setGrammar(String grammar) { + parameters.put(PARAM_GRAMMAR, toJsonString(grammar)); + return this; + } + + /** + * Override which part of the prompt is penalized for repetition. + * E.g. if original prompt is "Alice: Hello!" and penaltyPrompt is "Hello!", only the latter will be penalized if + * repeated. See pull request 3727 for more details. + */ + public InferenceParameters setPenaltyPrompt(String penaltyPrompt) { + parameters.put(PARAM_PENALTY_PROMPT, toJsonString(penaltyPrompt)); + return this; + } + + /** + * Override which tokens to penalize for repetition. + * E.g. if original prompt is "Alice: Hello!" and penaltyPrompt corresponds to the token ids of "Hello!", only the + * latter will be penalized if repeated. + * See pull request 3727 for more details. + */ + public InferenceParameters setPenaltyPrompt(int[] tokens) { + if (tokens.length > 0) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + for (int i = 0; i < tokens.length; i++) { + builder.append(tokens[i]); + if (i < tokens.length - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_PENALTY_PROMPT, builder.toString()); + } + return this; + } + + /** + * Set whether to ignore end of stream token and continue generating (implies --logit-bias 2-inf) + */ + public InferenceParameters setIgnoreEos(boolean ignoreEos) { + parameters.put(PARAM_IGNORE_EOS, String.valueOf(ignoreEos)); + return this; + } + + /** + * Modify the likelihood of tokens appearing in the completion by their id. E.g., Map.of(15043, 1f) + * to increase the likelihood of token ' Hello', or a negative value to decrease it. + * Note, this method overrides any previous calls to + *
    + *
  • {@link #setTokenBias(Map)}
    • + *
  • {@link #disableTokens(Collection)}
    • + *
  • {@link #disableTokenIds(Collection)}}
    • + *
+ */ + public InferenceParameters setTokenIdBias(Map logitBias) { + if (!logitBias.isEmpty()) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Map.Entry entry : logitBias.entrySet()) { + Integer key = entry.getKey(); + Float value = entry.getValue(); + builder.append("[") + .append(key) + .append(", ") + .append(value) + .append("]"); + if (i++ < logitBias.size() - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_LOGIT_BIAS, builder.toString()); + } + return this; + } + + /** + * Set tokens to disable, this corresponds to {@link #setTokenIdBias(Map)} with a value of + * {@link Float#NEGATIVE_INFINITY}. + * Note, this method overrides any previous calls to + *
    + *
  • {@link #setTokenIdBias(Map)}
    • + *
  • {@link #setTokenBias(Map)}
    • + *
  • {@link #disableTokens(Collection)}
    • + *
+ */ + public InferenceParameters disableTokenIds(Collection tokenIds) { + if (!tokenIds.isEmpty()) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Integer token : tokenIds) { + builder.append("[") + .append(token) + .append(", ") + .append(false) + .append("]"); + if (i++ < tokenIds.size() - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_LOGIT_BIAS, builder.toString()); + } + return this; + } + + /** + * Modify the likelihood of tokens appearing in the completion by their id. E.g., Map.of(" Hello", 1f) + * to increase the likelihood of token id 15043, or a negative value to decrease it. + * Note, this method overrides any previous calls to + *
    + *
  • {@link #setTokenIdBias(Map)}
    • + *
  • {@link #disableTokens(Collection)}
    • + *
  • {@link #disableTokenIds(Collection)}}
    • + *
+ */ + public InferenceParameters setTokenBias(Map logitBias) { + if (!logitBias.isEmpty()) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Map.Entry entry : logitBias.entrySet()) { + String key = entry.getKey(); + Float value = entry.getValue(); + builder.append("[") + .append(toJsonString(key)) + .append(", ") + .append(value) + .append("]"); + if (i++ < logitBias.size() - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_LOGIT_BIAS, builder.toString()); + } + return this; + } + + /** + * Set tokens to disable, this corresponds to {@link #setTokenBias(Map)} with a value of + * {@link Float#NEGATIVE_INFINITY}. + * Note, this method overrides any previous calls to + *
    + *
  • {@link #setTokenBias(Map)}
    • + *
  • {@link #setTokenIdBias(Map)}
    • + *
  • {@link #disableTokenIds(Collection)}
    • + *
+ */ + public InferenceParameters disableTokens(Collection tokens) { + if (!tokens.isEmpty()) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (String token : tokens) { + builder.append("[") + .append(toJsonString(token)) + .append(", ") + .append(false) + .append("]"); + if (i++ < tokens.size() - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_LOGIT_BIAS, builder.toString()); + } + return this; + } + + /** + * Set strings upon seeing which token generation is stopped + */ + public InferenceParameters setStopStrings(String... stopStrings) { + if (stopStrings.length > 0) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + for (int i = 0; i < stopStrings.length; i++) { + builder.append(toJsonString(stopStrings[i])); + if (i < stopStrings.length - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_STOP, builder.toString()); + } + return this; + } + + /** + * Set which samplers to use for token generation in the given order + */ + public InferenceParameters setSamplers(Sampler... samplers) { + if (samplers.length > 0) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + for (int i = 0; i < samplers.length; i++) { + switch (samplers[i]) { + case TOP_K: + builder.append("\"top_k\""); + break; + case TOP_P: + builder.append("\"top_p\""); + break; + case MIN_P: + builder.append("\"min_p\""); + break; + case TEMPERATURE: + builder.append("\"temperature\""); + break; + } + if (i < samplers.length - 1) { + builder.append(", "); + } + } + builder.append("]"); + parameters.put(PARAM_SAMPLERS, builder.toString()); + } + return this; + } + + /** + * Set whether generate should apply a chat template (default: false) + */ + public InferenceParameters setUseChatTemplate(boolean useChatTemplate) { + parameters.put(PARAM_USE_JINJA, String.valueOf(useChatTemplate)); + return this; + } + + /** + * Set the messages for chat-based inference. + * - Allows **only one** system message. + * - Allows **one or more** user/assistant messages. + */ + public InferenceParameters setMessages(String systemMessage, List> messages) { + StringBuilder messagesBuilder = new StringBuilder(); + messagesBuilder.append("["); + + // Add system message (if provided) + if (systemMessage != null && !systemMessage.isEmpty()) { + messagesBuilder.append("{\"role\": \"system\", \"content\": ") + .append(toJsonString(systemMessage)) + .append("}"); + if (!messages.isEmpty()) { + messagesBuilder.append(", "); + } + } + + // Add user/assistant messages + for (int i = 0; i < messages.size(); i++) { + Pair message = messages.get(i); + String role = message.getKey(); + String content = message.getValue(); + + if (!role.equals("user") && !role.equals("assistant")) { + throw new IllegalArgumentException("Invalid role: " + role + ". Role must be 'user' or 'assistant'."); + } + + messagesBuilder.append("{\"role\":") + .append(toJsonString(role)) + .append(", \"content\": ") + .append(toJsonString(content)) + .append("}"); + + if (i < messages.size() - 1) { + messagesBuilder.append(", "); + } + } + + messagesBuilder.append("]"); + + // Convert ArrayNode to a JSON string and store it in parameters + parameters.put(PARAM_MESSAGES, messagesBuilder.toString()); + return this; + } + + InferenceParameters setStream(boolean stream) { + parameters.put(PARAM_STREAM, String.valueOf(stream)); + return this; + } + +} diff --git a/src/main/java/de/kherud/llama/JsonParameters.java b/src/main/java/de/kherud/llama/JsonParameters.java new file mode 100644 index 000000000..e9916976c --- /dev/null +++ b/src/main/java/de/kherud/llama/JsonParameters.java @@ -0,0 +1,95 @@ +package de.kherud.llama; + +import java.util.HashMap; +import java.util.Map; + +/** + * The Java library re-uses most of the llama.cpp server code, which mostly works with JSONs. Thus, the complexity and + * maintainability is much lower if we work with JSONs. This class provides a simple abstraction to easily create + * JSON object strings by filling a Map<String, String> with key value pairs. + */ +abstract class JsonParameters { + + // We save parameters directly as a String map here, to re-use as much as possible of the (json-based) C++ code. + // The JNI code for a proper Java-typed data object is comparatively too complex and hard to maintain. + final Map parameters = new HashMap<>(); + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + int i = 0; + for (Map.Entry entry : parameters.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + builder.append("\t\"") + .append(key) + .append("\": ") + .append(value); + if (i++ < parameters.size() - 1) { + builder.append(","); + } + builder.append("\n"); + } + builder.append("}"); + return builder.toString(); + } + + // taken from org.json.JSONObject#quote(String, Writer) + String toJsonString(String text) { + if (text == null) return null; + StringBuilder builder = new StringBuilder((text.length()) + 2); + + char b; + char c = 0; + String hhhh; + int i; + int len = text.length(); + + builder.append('"'); + for (i = 0; i < len; i += 1) { + b = c; + c = text.charAt(i); + switch (c) { + case '\\': + case '"': + builder.append('\\'); + builder.append(c); + break; + case '/': + if (b == '<') { + builder.append('\\'); + } + builder.append(c); + break; + case '\b': + builder.append("\\b"); + break; + case '\t': + builder.append("\\t"); + break; + case '\n': + builder.append("\\n"); + break; + case '\f': + builder.append("\\f"); + break; + case '\r': + builder.append("\\r"); + break; + default: + if (c < ' ' || (c >= '\u0080' && c < '\u00a0') || (c >= '\u2000' && c < '\u2100')) { + builder.append("\\u"); + hhhh = Integer.toHexString(c); + builder.append("0000", 0, 4 - hhhh.length()); + builder.append(hhhh); + } + else { + builder.append(c); + } + } + } + builder.append('"'); + return builder.toString(); + } +} diff --git a/src/main/java/de/kherud/llama/LlamaException.java b/src/main/java/de/kherud/llama/LlamaException.java new file mode 100644 index 000000000..84d4ee7c3 --- /dev/null +++ b/src/main/java/de/kherud/llama/LlamaException.java @@ -0,0 +1,9 @@ +package de.kherud.llama; + +class LlamaException extends RuntimeException { + + public LlamaException(String message) { + super(message); + } + +} diff --git a/src/main/java/de/kherud/llama/LlamaGrammar.java b/src/main/java/de/kherud/llama/LlamaGrammar.java deleted file mode 100644 index f710b0463..000000000 --- a/src/main/java/de/kherud/llama/LlamaGrammar.java +++ /dev/null @@ -1,494 +0,0 @@ -package de.kherud.llama; - -import com.sun.jna.Memory; -import com.sun.jna.Native; -import com.sun.jna.Pointer; -import de.kherud.llama.foreign.LlamaLibrary; -import de.kherud.llama.foreign.NativeSize; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static de.kherud.llama.foreign.LlamaLibrary.llama_gretype.*; - -/** - * Implements a parser for an extended Backus-Naur form (BNF), producing the binary context-free grammar format - * specified by llama.h. Supports character ranges, grouping, and repetition operators. As an example, a grammar for - * arithmetic might look like: - * 
- * root  ::= expr
- * expr  ::= term ([-+/] term)
- * term  ::= num | "(" space expr ")" space
- * num   ::= [0-9]+ space
- * space ::= [ \t\n]*
- *
- * This class is implemented as an {@link AutoCloseable} in order to free the natively allocated memory for this - * grammar, after it is no longer needed. - */ -public class LlamaGrammar implements AutoCloseable { - - private final ParseState state; - final LlamaLibrary.llama_grammar foreign; - - /** - * Loads a grammar from a file, directly parses it, and allocates a native {@link LlamaLibrary.llama_grammar}. - * - * @param file the file containing a grammar - * @throws IOException on problems reading the file - */ - public LlamaGrammar(File file) throws IOException { - this(file.toPath()); - } - - /** - * Loads a grammar from a path, directly parses it, and allocates a native {@link LlamaLibrary.llama_grammar}. - * - * @param path the path to a file containing a grammar - * @throws IOException on problems reading the file - */ - public LlamaGrammar(Path path) throws IOException { - this(Files.readString(path, StandardCharsets.UTF_8)); - } - - /** - * Parses a grammar given as string and allocates a native {@link LlamaLibrary.llama_grammar}. - * - * @param grammar the grammar string - */ - public LlamaGrammar(String grammar) { - this.state = new ParseState(grammar); - this.foreign = state.create(); - } - - @Override - public String toString() { - return state.toString(); - } - - @Override - public void close() { - LlamaLibrary.llama_grammar_free(foreign); - } - - /** - * The state object used for parsing a grammar. - * It can't be modified and is used to allocate the native {@link LlamaLibrary.llama_grammar}. - */ - static final class ParseState { - - final Map symbol2Id = new HashMap<>(); - final Map id2Symbol = new HashMap<>(); - final List>> rules = new ArrayList<>(); - private String string; - - ParseState(String grammar) { - parse(grammar.getBytes(StandardCharsets.UTF_8)); - } - - private void parse(byte[] src) { - int pos = parseSpace(src, 0, true); - while (pos < src.length) { - pos = parseRule(src, pos); - } - } - - private LlamaLibrary.llama_grammar create() { - // creating the grammar is a bit tricky, since we need to allocate contiguous memory segments for the - // elements of each rule - - List rulePointers = new ArrayList<>(); - for (int ruleIndex = 0; ruleIndex < rules.size(); ruleIndex++) { - List> rule = rules.get(ruleIndex); - if (rule == null || rule.isEmpty()) { - throw new IllegalArgumentException("no rule for symbol '" + id2Symbol.get(ruleIndex) + "' found"); - } - Memory memory = new Memory((long) rule.size() * 8); - for (int i = 0; i < rule.size(); i++) { - Pair element = rule.get(i); - memory.setInt((long) i * 8, element.a); - memory.setInt((long) i * 8 + 4, element.b); - } - rulePointers.add(memory); - } - - Pointer memory = new Memory((long) Native.POINTER_SIZE * rulePointers.size()); - for (int i = 0; i < rulePointers.size(); i++) { - memory.setPointer((long) i * Native.POINTER_SIZE, rulePointers.get(i)); - } - - NativeSize nRules = new NativeSize(rulePointers.size()); - NativeSize startRuleIndex = new NativeSize(symbol2Id.get("root")); - - return LlamaLibrary.llama_grammar_init(memory, nRules, startRuleIndex); - } - - private int parseSpace(byte[] src, int pos, boolean newlineOk) { - while (pos < src.length) { - byte currentChar = src[pos]; - if (currentChar == ' ' - || currentChar == '\t' - || currentChar == '#' - || (newlineOk && (currentChar == '\r' || currentChar == '\n'))) { - if (currentChar == '#') { - while (pos < src.length - && src[pos] != '\r' - && src[pos] != '\n') { - pos++; - } - } else { - pos++; - } - } else { - break; - } - } - return pos; - } - - private int parseRule(byte[] src, int startPos) throws RuntimeException { - int nameEnd = parseName(src, startPos); - int pos = parseSpace(src, nameEnd, false); - int nameLen = nameEnd - startPos; - - int ruleId = getSymbolId(src, startPos, nameLen); - String name = new String(src, startPos, nameLen, StandardCharsets.UTF_8); - - if (src.length <= pos + 2 || !(src[pos] == ':' && src[pos + 1] == ':' && src[pos + 2] == '=')) { - throw new RuntimeException("Expecting ::= at position " + pos); - } - pos = parseSpace(src, pos + 3, true); - - pos = parseAlternates(src, pos, name, ruleId, false); - - if (pos < src.length) { - if (src[pos] == '\r') { - pos += (pos + 1 < src.length && src[pos + 1] == '\n') ? 2 : 1; - } else if (src[pos] == '\n') { - pos++; - } else { - throw new RuntimeException("Expecting newline or end at position " + pos); - } - } - - return parseSpace(src, pos, true); - } - - private int parseName(byte[] src, int startPos) { - int pos = startPos; - while (pos < src.length && isWordChar(src[pos])) { - pos++; - } - if (pos == startPos) { - throw new RuntimeException("Expecting name at position " + startPos); - } - return pos; - } - - private boolean isWordChar(byte c) { - return 'a' <= c && c <= 'z' - || 'A' <= c && c <= 'Z' - || c == '-' - || ('0' <= c && c <= '9'); - } - - private int getSymbolId(byte[] src, int pos, int len) { - String str = new String(src, pos, len, StandardCharsets.UTF_8); - if (!symbol2Id.containsKey(str)) { - int nextId = symbol2Id.size(); - symbol2Id.put(str, nextId); - id2Symbol.put(nextId, str); - } - return symbol2Id.get(str); - } - - private int parseAlternates(byte[] src, int startPos, String ruleName, int ruleId, boolean isNested) { - List> rule = new ArrayList<>(); - int pos = parseSequence(ruleName, src, startPos, rule, isNested); - - while (pos < src.length && src[pos] == '|') { - rule.add(new Pair<>(LLAMA_GRETYPE_ALT, 0)); - pos = parseSpace(src, pos + 1, true); - pos = parseSequence(ruleName, src, pos, rule, isNested); - } - - rule.add(new Pair<>(LLAMA_GRETYPE_END, 0)); - addRule(ruleId, rule); - - return pos; - } - - private void addRule(int ruleId, List> rule) { - // resize the rules ArrayList if necessary - while (rules.size() <= ruleId) { - rules.add(null); - } - rules.set(ruleId, rule); - } - - private int parseSequence(String ruleName, byte[] src, int startPos, List> outElements, boolean isNested) { - int lastSymStart = outElements.size(); - int pos = startPos; - - while (pos < src.length) { - if (src[pos] == '"') { - pos++; - lastSymStart = outElements.size(); - while (src[pos] != '"') { - Pair charPair = parseChar(src, pos); - pos = charPair.b; - outElements.add(new Pair<>(LLAMA_GRETYPE_CHAR, charPair.a)); - } - pos = parseSpace(src, pos + 1, isNested); - } else if (src[pos] == '[') { // char range(s) - pos++; - int startType = LLAMA_GRETYPE_CHAR; - if (src[pos] == '^') { - pos++; - startType = LLAMA_GRETYPE_CHAR_NOT; - } - lastSymStart = outElements.size(); - while (src[pos] != ']') { - Pair charPair = parseChar(src, pos); - pos = charPair.b; - int type = lastSymStart < outElements.size() ? LLAMA_GRETYPE_CHAR_ALT : startType; - - outElements.add(new Pair<>(type, charPair.a)); - if (src[pos] == '-' && src[pos + 1] != ']') { - Pair endCharPair = parseChar(src, pos + 1); - pos = endCharPair.b; - outElements.add(new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, endCharPair.a)); - } - } - pos = parseSpace(src, pos + 1, isNested); - } else if (isWordChar(src[pos])) { // rule reference - int nameEnd = parseName(src, pos); - int refRuleId = getSymbolId(src, pos, nameEnd - pos); - pos = parseSpace(src, nameEnd, isNested); - lastSymStart = outElements.size(); - outElements.add(new Pair<>(LLAMA_GRETYPE_RULE_REF, refRuleId)); - } else if (src[pos] == '(') { // grouping - pos = parseSpace(src, pos + 1, true); - int subRuleId = generateSymbolId(ruleName); - pos = parseAlternates(src, pos, ruleName, subRuleId, true); - lastSymStart = outElements.size(); - outElements.add(new Pair<>(LLAMA_GRETYPE_RULE_REF, subRuleId)); - if (src[pos] != ')') { - throw new RuntimeException("expecting ')' at " + pos); - } - pos = parseSpace(src, pos + 1, isNested); - } else if (src[pos] == '*' || src[pos] == '+' || src[pos] == '?') { - if (lastSymStart == outElements.size()) { - throw new RuntimeException("expecting preceding item to */+/? at " + pos); - } - // apply transformation to previous symbol (last_sym_start to end) according to - // rewrite rules: - // S* --> S' ::= S S' | - // S+ --> S' ::= S S' | S - // S? --> S' ::= S | - int subRuleId = generateSymbolId(ruleName); - // Add preceding symbol to generated rule - List> subRule = new ArrayList<>(outElements.subList(lastSymStart, outElements.size())); - if (src[pos] == '*' || src[pos] == '+') { - // Cause generated rule to recurse - subRule.add(new Pair<>(LLAMA_GRETYPE_RULE_REF, subRuleId)); - } - // Mark start of alternate def - subRule.add(new Pair<>(LLAMA_GRETYPE_ALT, 0)); - if (src[pos] == '+') { - // Add preceding symbol as alternate only for '+' (otherwise empty) - subRule.addAll(outElements.subList(lastSymStart, outElements.size())); - } - subRule.add(new Pair<>(LLAMA_GRETYPE_END, 0)); - addRule(subRuleId, subRule); - - // In original rule, replace previous symbol with reference to generated rule - outElements.subList(lastSymStart, outElements.size()).clear(); - outElements.add(new Pair<>(LLAMA_GRETYPE_RULE_REF, subRuleId)); - - pos = parseSpace(src, pos + 1, isNested); - } else { - break; - } - } - return pos; - } - - private Pair parseChar(byte[] src, int pos) throws RuntimeException { - if (src[pos] == '\\') { - switch (src[pos + 1]) { - case 'x': - return parseHex(src, pos + 2, 2); - case 'u': - return parseHex(src, pos + 2, 4); - case 'U': - return parseHex(src, pos + 2, 8); - case 't': - return new Pair<>((int) '\t', pos + 2); - case 'r': - return new Pair<>((int) '\r', pos + 2); - case 'n': - return new Pair<>((int) '\n', pos + 2); - case '\\': - case '"': - case '[': - case ']': - return new Pair<>((int) src[pos + 1], pos + 2); - default: - throw new RuntimeException("Unknown escape at " + pos); - } - } else { - return decodeUtf8(src, pos); - } - } - - private Pair parseHex(byte[] src, int pos, int size) throws RuntimeException { - int end = pos + size; - int value = 0; - for (; pos < end && src[pos] != 0; pos++) { - value <<= 4; - byte c = src[pos]; - if ('a' <= c && c <= 'f') { - value += c - 'a' + 10; - } else if ('A' <= c && c <= 'F') { - value += c - 'A' + 10; - } else if ('0' <= c && c <= '9') { - value += c - '0'; - } else { - break; - } - } - if (pos != end) { - throw new RuntimeException("Expecting " + size + " hex chars at " + pos); - } - return new Pair<>(value, pos); - } - - private Pair decodeUtf8(byte[] src, int pos) { - final int[] lookup = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4}; - byte firstByte = src[pos]; - byte highBits = (byte) (firstByte >> 4); - int len = lookup[highBits]; - byte mask = (byte) ((1 << (8 - len)) - 1); - int value = firstByte & mask; - int end = pos + len; - pos = pos + 1; - for (; pos < end && src[pos] != 0; pos++) { - value = (value << 6) + (src[pos] & 0x3F); - } - return new Pair<>(value, pos); - } - - private int generateSymbolId(String baseName) { - int nextId = symbol2Id.size(); - symbol2Id.put(baseName + "_" + nextId, nextId); - return nextId; - } - - @Override - public String toString() { - if (string != null) { - return string; - } - StringBuilder grammarBuilder = new StringBuilder(); - for (int i = 0, end = rules.size(); i < end; i++) { - grammarBuilder.append(i).append(": "); - List> rule = rules.get(i); - appendRule(grammarBuilder, i, rule); - } - string = grammarBuilder.toString(); - return string; - } - - private void appendRule(StringBuilder builder, int ruleId, List> rule) { - if (rule.isEmpty() || rule.get(rule.size() - 1).a != LLAMA_GRETYPE_END) { - throw new RuntimeException("Malformed rule, does not end with LLAMA_GRETYPE_END: " + ruleId); - } - builder.append(id2Symbol.get(ruleId)).append(" ::= "); - for (int i = 0, end = rule.size() - 1; i < end; i++) { - Pair elem = rule.get(i); - switch (elem.a) { - case LLAMA_GRETYPE_END: - throw new RuntimeException("Unexpected end of rule: " + ruleId + "," + i); - case LLAMA_GRETYPE_ALT: - builder.append("| "); - break; - case LLAMA_GRETYPE_RULE_REF: - builder.append(id2Symbol.get(elem.b)).append(" "); - break; - case LLAMA_GRETYPE_CHAR: - builder.append("["); - printGrammarChar(builder, elem.b); - break; - case LLAMA_GRETYPE_CHAR_NOT: - builder.append("[^"); - printGrammarChar(builder, elem.b); - break; - case LLAMA_GRETYPE_CHAR_RNG_UPPER: - if (i == 0 || !isCharElement(rule.get(i - 1))) { - throw new RuntimeException( - "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + ruleId + "," + i); - } - builder.append("-"); - printGrammarChar(builder, elem.b); - break; - case LLAMA_GRETYPE_CHAR_ALT: - if (i == 0 || !isCharElement(rule.get(i - 1))) { - throw new RuntimeException( - "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + ruleId + "," + i); - } - printGrammarChar(builder, elem.b); - break; - default: - throw new RuntimeException("Unknown type: " + elem.b); - } - if (isCharElement(elem)) { - switch (rule.get(i + 1).b) { - case LLAMA_GRETYPE_CHAR_ALT: - case LLAMA_GRETYPE_CHAR_RNG_UPPER: - break; - default: - builder.append("] "); - } - } - } - builder.append("\n"); - } - - private void printGrammarChar(StringBuilder builder, int c) { - if (c >= 0x20 && c <= 0x7f) { - builder.append((char) c); - } else { - builder.append(String.format("", c)); - } - } - - private boolean isCharElement(Pair element) { - switch (element.a) { - case LLAMA_GRETYPE_CHAR: - case LLAMA_GRETYPE_CHAR_NOT: - case LLAMA_GRETYPE_CHAR_ALT: - case LLAMA_GRETYPE_CHAR_RNG_UPPER: - return true; - default: - return false; - } - } - } - - static final class Pair { - final A a; - final B b; - - Pair(A a, B b) { - this.a = a; - this.b = b; - } - } -} diff --git a/src/main/java/de/kherud/llama/LlamaIterable.java b/src/main/java/de/kherud/llama/LlamaIterable.java new file mode 100644 index 000000000..7e6dff89a --- /dev/null +++ b/src/main/java/de/kherud/llama/LlamaIterable.java @@ -0,0 +1,15 @@ +package de.kherud.llama; + +import org.jetbrains.annotations.NotNull; + +/** + * An iterable used by {@link LlamaModel#generate(InferenceParameters)} that specifically returns a {@link LlamaIterator}. + */ +@FunctionalInterface +public interface LlamaIterable extends Iterable { + + @NotNull + @Override + LlamaIterator iterator(); + +} diff --git a/src/main/java/de/kherud/llama/LlamaIterator.java b/src/main/java/de/kherud/llama/LlamaIterator.java new file mode 100644 index 000000000..cb1c5c2ca --- /dev/null +++ b/src/main/java/de/kherud/llama/LlamaIterator.java @@ -0,0 +1,51 @@ +package de.kherud.llama; + +import java.lang.annotation.Native; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * This iterator is used by {@link LlamaModel#generate(InferenceParameters)}. In addition to implementing {@link Iterator}, + * it allows to cancel ongoing inference (see {@link #cancel()}). + */ +public final class LlamaIterator implements Iterator { + + private final LlamaModel model; + private final int taskId; + + @Native + @SuppressWarnings("FieldMayBeFinal") + private boolean hasNext = true; + + LlamaIterator(LlamaModel model, InferenceParameters parameters) { + this.model = model; + parameters.setStream(true); + taskId = model.requestCompletion(parameters.toString()); + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public LlamaOutput next() { + if (!hasNext) { + throw new NoSuchElementException(); + } + LlamaOutput output = model.receiveCompletion(taskId); + hasNext = !output.stop; + if (output.stop) { + model.releaseTask(taskId); + } + return output; + } + + /** + * Cancel the ongoing generation process. + */ + public void cancel() { + model.cancelCompletion(taskId); + hasNext = false; + } +} diff --git a/src/main/java/de/kherud/llama/LlamaLoader.java b/src/main/java/de/kherud/llama/LlamaLoader.java new file mode 100644 index 000000000..586925229 --- /dev/null +++ b/src/main/java/de/kherud/llama/LlamaLoader.java @@ -0,0 +1,272 @@ +/*-------------------------------------------------------------------------- + * Copyright 2007 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ + +package de.kherud.llama; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import org.jetbrains.annotations.Nullable; + +/** + * Set the system properties, de.kherud.llama.lib.path, de.kherud.llama.lib.name, appropriately so that the + * library can find *.dll, *.dylib and *.so files, according to the current OS (win, linux, mac). + * + *

The library files are automatically extracted from this project's package (JAR). + * + *

usage: call {@link #initialize()} before using the library. + * + * @author leo + */ +@SuppressWarnings("UseOfSystemOutOrSystemErr") +class LlamaLoader { + + private static boolean extracted = false; + + /** + * Loads the llama and jllama shared libraries + */ + static synchronized void initialize() throws UnsatisfiedLinkError { + // only cleanup before the first extract + if (!extracted) { + cleanup(); + } + if ("Mac".equals(OSInfo.getOSName())) { + String nativeDirName = getNativeResourcePath(); + String tempFolder = getTempDir().getAbsolutePath(); + System.out.println(nativeDirName); + Path metalFilePath = extractFile(nativeDirName, "ggml-metal.metal", tempFolder, false); + if (metalFilePath == null) { + System.err.println("'ggml-metal.metal' not found"); + } + } + loadNativeLibrary("jllama"); + extracted = true; + } + + /** + * Deleted old native libraries e.g. on Windows the DLL file is not removed on VM-Exit (bug #80) + */ + private static void cleanup() { + try (Stream dirList = Files.list(getTempDir().toPath())) { + dirList.filter(LlamaLoader::shouldCleanPath).forEach(LlamaLoader::cleanPath); + } + catch (IOException e) { + System.err.println("Failed to open directory: " + e.getMessage()); + } + } + + private static boolean shouldCleanPath(Path path) { + String fileName = path.getFileName().toString(); + return fileName.startsWith("jllama") || fileName.startsWith("llama"); + } + + private static void cleanPath(Path path) { + try { + Files.delete(path); + } + catch (Exception e) { + System.err.println("Failed to delete old native lib: " + e.getMessage()); + } + } + + private static void loadNativeLibrary(String name) { + List triedPaths = new LinkedList<>(); + + String nativeLibName = System.mapLibraryName(name); + String nativeLibPath = System.getProperty("de.kherud.llama.lib.path"); + if (nativeLibPath != null) { + Path path = Paths.get(nativeLibPath, nativeLibName); + if (loadNativeLibrary(path)) { + return; + } + else { + triedPaths.add(nativeLibPath); + } + } + + if (OSInfo.isAndroid()) { + try { + // loadLibrary can load directly from packed apk file automatically + // if java-llama.cpp is added as code source + System.loadLibrary(name); + return; + } + catch (UnsatisfiedLinkError e) { + triedPaths.add("Directly from .apk/lib"); + } + } + + // Try to load the library from java.library.path + String javaLibraryPath = System.getProperty("java.library.path", ""); + for (String ldPath : javaLibraryPath.split(File.pathSeparator)) { + if (ldPath.isEmpty()) { + continue; + } + Path path = Paths.get(ldPath, nativeLibName); + if (loadNativeLibrary(path)) { + return; + } + else { + triedPaths.add(ldPath); + } + } + + // As a last resort try load the os-dependent library from the jar file + nativeLibPath = getNativeResourcePath(); + if (hasNativeLib(nativeLibPath, nativeLibName)) { + // temporary library folder + String tempFolder = getTempDir().getAbsolutePath(); + // Try extracting the library from jar + if (extractAndLoadLibraryFile(nativeLibPath, nativeLibName, tempFolder)) { + return; + } + else { + triedPaths.add(nativeLibPath); + } + } + + throw new UnsatisfiedLinkError( + String.format( + "No native library found for os.name=%s, os.arch=%s, paths=[%s]", + OSInfo.getOSName(), + OSInfo.getArchName(), + String.join(File.pathSeparator, triedPaths) + ) + ); + } + + /** + * Loads native library using the given path and name of the library + * + * @param path path of the native library + * @return true for successfully loading, otherwise false + */ + public static boolean loadNativeLibrary(Path path) { + if (!Files.exists(path)) { + return false; + } + String absolutePath = path.toAbsolutePath().toString(); + try { + System.load(absolutePath); + return true; + } + catch (UnsatisfiedLinkError e) { + System.err.println(e.getMessage()); + System.err.println("Failed to load native library: " + absolutePath + ". osinfo: " + OSInfo.getNativeLibFolderPathForCurrentOS()); + return false; + } + } + + @Nullable + private static Path extractFile(String sourceDirectory, String fileName, String targetDirectory, boolean addUuid) { + String nativeLibraryFilePath = sourceDirectory + "/" + fileName; + + Path extractedFilePath = Paths.get(targetDirectory, fileName); + + try { + // Extract a native library file into the target directory + try (InputStream reader = LlamaLoader.class.getResourceAsStream(nativeLibraryFilePath)) { + if (reader == null) { + return null; + } + Files.copy(reader, extractedFilePath, StandardCopyOption.REPLACE_EXISTING); + } + finally { + // Delete the extracted lib file on JVM exit. + extractedFilePath.toFile().deleteOnExit(); + } + + // Set executable (x) flag to enable Java to load the native library + extractedFilePath.toFile().setReadable(true); + extractedFilePath.toFile().setWritable(true, true); + extractedFilePath.toFile().setExecutable(true); + + // Check whether the contents are properly copied from the resource folder + try (InputStream nativeIn = LlamaLoader.class.getResourceAsStream(nativeLibraryFilePath); + InputStream extractedLibIn = Files.newInputStream(extractedFilePath)) { + if (!contentsEquals(nativeIn, extractedLibIn)) { + throw new RuntimeException(String.format("Failed to write a native library file at %s", extractedFilePath)); + } + } + + System.out.println("Extracted '" + fileName + "' to '" + extractedFilePath + "'"); + return extractedFilePath; + } + catch (IOException e) { + System.err.println(e.getMessage()); + return null; + } + } + + /** + * Extracts and loads the specified library file to the target folder + * + * @param libFolderForCurrentOS Library path. + * @param libraryFileName Library name. + * @param targetFolder Target folder. + * @return whether the library was successfully loaded + */ + private static boolean extractAndLoadLibraryFile(String libFolderForCurrentOS, String libraryFileName, String targetFolder) { + Path path = extractFile(libFolderForCurrentOS, libraryFileName, targetFolder, true); + if (path == null) { + return false; + } + return loadNativeLibrary(path); + } + + private static boolean contentsEquals(InputStream in1, InputStream in2) throws IOException { + if (!(in1 instanceof BufferedInputStream)) { + in1 = new BufferedInputStream(in1); + } + if (!(in2 instanceof BufferedInputStream)) { + in2 = new BufferedInputStream(in2); + } + + int ch = in1.read(); + while (ch != -1) { + int ch2 = in2.read(); + if (ch != ch2) { + return false; + } + ch = in1.read(); + } + int ch2 = in2.read(); + return ch2 == -1; + } + + private static File getTempDir() { + return new File(System.getProperty("de.kherud.llama.tmpdir", System.getProperty("java.io.tmpdir"))); + } + + private static String getNativeResourcePath() { + String packagePath = LlamaLoader.class.getPackage().getName().replace(".", "/"); + return String.format("/%s/%s", packagePath, OSInfo.getNativeLibFolderPathForCurrentOS()); + } + + private static boolean hasNativeLib(String path, String libraryName) { + return LlamaLoader.class.getResource(path + "/" + libraryName) != null; + } +} diff --git a/src/main/java/de/kherud/llama/LlamaModel.java b/src/main/java/de/kherud/llama/LlamaModel.java index ae734fb42..eab36202c 100644 --- a/src/main/java/de/kherud/llama/LlamaModel.java +++ b/src/main/java/de/kherud/llama/LlamaModel.java @@ -1,18 +1,13 @@ package de.kherud.llama; -import com.sun.jna.Pointer; -import de.kherud.llama.foreign.LlamaLibrary; -import de.kherud.llama.foreign.NativeSize; -import de.kherud.llama.foreign.llama_timings; -import de.kherud.llama.foreign.llama_token_data; -import de.kherud.llama.foreign.llama_token_data_array; -import org.jetbrains.annotations.NotNull; +import de.kherud.llama.args.LogFormat; import org.jetbrains.annotations.Nullable; -import java.nio.IntBuffer; +import java.lang.annotation.Native; import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.NoSuchElementException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.function.BiConsumer; /** @@ -20,570 +15,157 @@ * Upon being created, it natively allocates memory for the model context. * Thus, this class is an {@link AutoCloseable}, in order to de-allocate the memory when it is no longer being needed. *

- * This class is NOT stateless, it internally manages the state of a single conversation. - * If this state isn't needed or wanted, call {@link #reset()}. - *

* The main functionality of this class is: *

    - *
  • Streaming answers (and probabilities) via {@link #generate(String)}
    • - *
  • Creating whole responses to prompts via {@link #complete(String)}
    • - *
  • Creating embeddings via {@link #getEmbedding(String)} (make sure to configure {@link Parameters.Builder#setEmbedding(boolean)}
    • + *
  • Streaming answers (and probabilities) via {@link #generate(InferenceParameters)}
    • + *
  • Creating whole responses to prompts via {@link #complete(InferenceParameters)}
    • + *
  • Creating embeddings via {@link #embed(String)} (make sure to configure {@link ModelParameters#enableEmbedding()}
    • *
  • Accessing the tokenizer via {@link #encode(String)} and {@link #decode(int[])}
    • *
*/ public class LlamaModel implements AutoCloseable { - static { - LlamaLibrary.llama_backend_init((byte) 0); - } - private static BiConsumer logCallback; - - private final Parameters params; - private final LlamaLibrary.llama_model model; - private final LlamaLibrary.llama_context ctx; - - // cache some things for performance - private final Pointer logitsPointer; // pointer to retrieve the logits of the llm - private final SliceableIntBuffer contextBuffer; // used to hold all tokens of a conversation - private final SliceableIntBuffer tokenBuffer; // used for tokenization - private final byte[] tokenPieceBuffer; // used to decode tokens to string - private final llama_token_data.ByReference[] candidateData; // candidates used for sampling - private final llama_token_data_array candidates; // array holding the candidates - private final int nVocab; - private final int tokenBos; - private final int tokenEos; - private final int tokenNl; - private int nPast = 0; // how many evaluated tokens are currently in the context buffer - private int nContext = 0; // how many tokens currently are in the context buffer - - /** - * Load a gguf llama.cpp model from a given file path with default {@link Parameters}. - * - * @param filePath a file path pointing to the model - */ - public LlamaModel(String filePath) { - this(filePath, new Parameters.Builder().build()); - } - - /** - * Load a gguf llama.cpp model from a given file path with custom {@link Parameters}. - * - * @param filePath a file path pointing to the model - * @param params the set of previously configured options - */ - public LlamaModel(String filePath, Parameters params) { - // load the model - this.params = params; - model = LlamaLibrary.llama_load_model_from_file(filePath, params.ctx); - if (model == null) { - throw new RuntimeException("error: unable to load model"); - } - ctx = LlamaLibrary.llama_new_context_with_model(model, params.ctx); - - // load lora adapter if configured - if (params.loraAdapter != null - && params.loraBase != null - && LlamaLibrary.llama_model_apply_lora_from_file(model, params.loraAdapter, params.loraBase, params.nThreads) != 0) { - throw new RuntimeException("error: unable to apply lora"); - } - - // setup some cached variables used throughout lifecycle - contextBuffer = new SliceableIntBuffer(IntBuffer.allocate(params.ctx.n_ctx)); - tokenBuffer = new SliceableIntBuffer(IntBuffer.allocate(params.ctx.n_ctx)); - tokenPieceBuffer = new byte[64]; - nVocab = getVocabularySize(); - - logitsPointer = LlamaLibrary.llama_get_logits(ctx).getPointer(); - candidateData = (llama_token_data.ByReference[]) new llama_token_data.ByReference().toArray(nVocab); - candidates = new llama_token_data_array(); - - tokenBos = LlamaLibrary.llama_token_bos(ctx); - tokenEos = LlamaLibrary.llama_token_eos(ctx); - tokenNl = LlamaLibrary.llama_token_nl(ctx); - } - - /** - * Generate and stream outputs. Unless {@link #reset()} is called, the previous conversation is used as context. - * Note, that the prompt isn't preprocessed in any way, nothing like "User: ", "###Instruction", etc. is added. - * - * @param prompt the LLM prompt - * @return iterable LLM outputs - */ - public Iterable generate(String prompt) { - return new Iterable<>() { - @NotNull - @Override - public Iterator iterator() { - return new LlamaIterator(prompt); - } - }; - } - - /** - * Generate and return a whole answer. Unless {@link #reset()} is called, the previous conversation is used as context. - * Note, that the prompt isn't preprocessed in any way, nothing like "User: ", "###Instruction", etc. is added. - * - * @param prompt the LLM prompt - * @return an LLM response - */ - public String complete(String prompt) { - StringBuilder builder = new StringBuilder(); - Iterator iterator = new LlamaIterator(prompt); - while (iterator.hasNext()) { - Output output = iterator.next(); - builder.append(output); - } - return builder.toString(); - } - - /** - * Get the embedding of a string. Unless {@link #reset()} is called, the previous conversation is used as context. - * Note, that the prompt isn't preprocessed in any way, nothing like "User: ", "###Instruction", etc. is added. - * - * @param prompt the string to embed - * @return an embedding float array the size of {@link #getEmbeddingSize()} - * @throws IllegalStateException if embedding mode was not activated (see {@link Parameters.Builder#setEmbedding(boolean)}) - */ - public float[] getEmbedding(String prompt) { - if (params.ctx.embedding == 0) { - throw new IllegalStateException("embedding mode not activated (see parameters)"); - } - SliceableIntBuffer tokens = tokenize(prompt, false); - addContext(tokens); - evaluate(); - return LlamaLibrary.llama_get_embeddings(ctx).getPointer().getFloatArray(0, getEmbeddingSize()); - } - - /** - * Resets the state of the LLM, its timings, and discards any previous conversation. - */ - public void reset() { - LlamaLibrary.llama_reset_timings(ctx); - this.contextBuffer.clear(); - this.nContext = 0; - this.nPast = 0; - } - - /** - * Tokenize a prompt given the native tokenizer - * - * @param prompt the prompt to tokenize - * @return an array of integers each representing a token id (see {@link #getVocabularySize()}) - */ - public int[] encode(String prompt) { - SliceableIntBuffer buffer = tokenize(prompt, false); - int[] tokens = new int[buffer.capacity()]; - System.arraycopy(buffer.delegate.array(), 0, tokens, 0, buffer.capacity()); - return tokens; - } - - /** - * Convert an array of token ids to its string representation - * - * @param tokens an array of tokens conforming to {@link #getVocabularySize()} - * @return the token ids decoded to a string - */ - public String decode(int[] tokens) { - StringBuilder builder = new StringBuilder(); - for (int token : tokens) { - String decoded = decodeToken(token); - builder.append(decoded); - } - return builder.toString(); - } - - /** - * Sets a callback for both Java and C++ log messages. - * Can be set to {@code null} to disable logging. - * Note, that in that case C++ output will appear in stdout/err, however. - * To completely silence output provide a callback like: - * 
-     * LlamaModel.setLogger((level, message) -> {});
-     *
- * - * @param logCallback a method to call for log messages - */ - public static void setLogger(@Nullable BiConsumer logCallback) { - LlamaModel.logCallback = logCallback; - // We currently do not allow to pass any user data to `llama_log_set`, since the JVM might move - // the object around in the memory, thus invalidating any pointers. - // Maybe this could be circumvented by allowing something like to be passed - // as user data. However, the use cases for this are minuscule I think. - if (logCallback == null) { - LlamaLibrary.llama_log_set(null, null); - } else { - LlamaLibrary.llama_log_set((code, text, user_data) -> { - LogLevel level = LogLevel.fromCode(code); - logCallback.accept(level, text); - }, null); - } - } - - /** - * Returns the context size of the loaded model, e.g., how many tokens the LLM can process at most in one request. - * E.g., for llama 1 this size is 2048. - * - * @return the context size of the loaded model - */ - public int getContextSize() { - return LlamaLibrary.llama_n_ctx(ctx); - } - - /** - * Returns the hidden dimensionality of the loaded model, which corresponds to the size of {@link #getEmbedding(String)}. - * This size typically depends on the amount of parameters in the model, e.g., for llama 2 13b this size is 5120. - * - * @return the amount of embedding dimensions - */ - public int getEmbeddingSize() { - return LlamaLibrary.llama_n_embd(ctx); - } - - /** - * Returns the total amount of tokens in the vocabulary. - * - * @return the vocabulary size - */ - public int getVocabularySize() { - return LlamaLibrary.llama_n_vocab(ctx); - } - - /** - * Returns the tokenization method / vocabulary type used by this model. - * - * @return the vocabulary type - */ - public VocabularyType getVocabularyType() { - int code = LlamaLibrary.llama_vocab_type(ctx); - return VocabularyType.fromCode(code); - } - - /** - * Returns the memory size of this model in bytes - * - * @return amount of bytes allocated - */ - public long getMemorySize() { - return LlamaLibrary.llama_model_size(model); - } - - /** - * Returns the amount of parameters this model has - * - * @return amount of parameters - */ - public long getAmountParameters() { - return LlamaLibrary.llama_model_n_params(model); - } - - /** - * Get performance information about this model - * - * @return the inference timings - */ - public llama_timings getTimings() { - return LlamaLibrary.llama_get_timings(ctx); - } - - /** - * Deallocates the native memory used by this model - */ - @Override - public void close() { - LlamaLibrary.llama_free_model(model); - LlamaLibrary.llama_free(ctx); - } - - @Override - public String toString() { - byte[] buffer = new byte[512]; - int size = LlamaLibrary.llama_model_desc(model, buffer, new NativeSize(buffer.length)); - return new String(buffer, 0, Math.min(size, buffer.length), StandardCharsets.UTF_8); - } - - - /** - * Internally tokenizes a prompt and returns its tokens without any padding. - * At most {@link #getContextSize()} tokens can be tokenized. Makes use of {@link #tokenBuffer}. - * - * @param prompt the prompt to tokenize - * @return an IntBuffer containing the tokenized prompt without any padding - * @throws RuntimeException if tokenization fails - */ - private SliceableIntBuffer tokenize(String prompt, boolean addBos) { - int nTokens = LlamaLibrary.llama_tokenize(ctx, prompt, tokenBuffer.delegate, params.ctx.n_ctx, addBos ? (byte) 1 : 0); - if (nTokens < 0) { - throw new RuntimeException("tokenization failed due to unknown reasons"); - } - return tokenBuffer.slice(0, nTokens); - } - - private void evaluate() { - while (nPast < nContext) { - int nEval = nContext - nPast; - if (nEval > params.ctx.n_batch) { - nEval = params.ctx.n_batch; - } - if (LlamaLibrary.llama_eval(ctx, contextBuffer.slice(nPast, nEval).delegate, nEval, nPast, params.nThreads) != 0) { - String msg = String.format("evaluation failed (%d to evaluate, %d past, %d threads)", nEval, nPast, params.nThreads); - log(LogLevel.ERROR, msg); - throw new RuntimeException("token evaluation failed"); - } - nPast += nEval; - } - } - - private Output sample() { - float[] logits = logitsPointer.getFloatArray(0, nVocab); - float nlLogit = logits[tokenNl]; - params.logitBias.forEach((i, bias) -> logits[i] += bias); - // I'm not sure why anything but `setLogit` has to be called here for `candidateData` and `candidates` again. - // Otherwise, the results are garbage, however. Maybe the JVM/JIT is moving stuff around. - for (int i = 0; i < nVocab; i++) { - candidateData[i].setId(i); - candidateData[i].setLogit(logits[i]); -// candidateData[i].setP(0f); - } - candidates.setData(candidateData[0]); - candidates.setSize(new NativeSize(nVocab)); - candidates.setSorted((byte) 0); - - samplePenalty(); - - if (!params.penalizeNl) { - candidateData[tokenNl].setLogit(nlLogit); - } - - if (params.grammar != null) { - LlamaLibrary.llama_sample_grammar(ctx, candidates, params.grammar.foreign); - } - - int token; - if (params.temperature == 0) { - token = sampleGreedy(); - } else if (params.mirostat == Parameters.MiroStat.V1) { - token = sampleMirostatV1(); - } else if (params.mirostat == Parameters.MiroStat.V2) { - token = sampleMirostatV2(); + static { + LlamaLoader.initialize(); + } + + @Native + private long ctx; + + /** + * Load with the given {@link ModelParameters}. Make sure to either set + *
    + *
  • {@link ModelParameters#setModel(String)}
    • + *
  • {@link ModelParameters#setModelUrl(String)}
    • + *
  • {@link ModelParameters#setHfRepo(String)}, {@link ModelParameters#setHfFile(String)}
    • + *
+ * + * @param parameters the set of options + * @throws LlamaException if no model could be loaded from the given file path + */ + public LlamaModel(ModelParameters parameters) { + loadModel(parameters.toArray()); + } + + /** + * Generate and return a whole answer with custom parameters. Note, that the prompt isn't preprocessed in any + * way, nothing like "User: ", "###Instruction", etc. is added. + * + * @return an LLM response + */ + public String complete(InferenceParameters parameters) { + parameters.setStream(false); + int taskId = requestCompletion(parameters.toString()); + LlamaOutput output = receiveCompletion(taskId); + return output.text; + } + + /** + * Generate and stream outputs with custom inference parameters. Note, that the prompt isn't preprocessed in any + * way, nothing like "User: ", "###Instruction", etc. is added. + * + * @return iterable LLM outputs + */ + public LlamaIterable generate(InferenceParameters parameters) { + return () -> new LlamaIterator(this, parameters); + } + + + + /** + * Get the embedding of a string. Note, that the prompt isn't preprocessed in any way, nothing like + * "User: ", "###Instruction", etc. is added. + * + * @param prompt the string to embed + * @return an embedding float array + * @throws IllegalStateException if embedding mode was not activated (see {@link ModelParameters#enableEmbedding()}) + */ + public native float[] embed(String prompt); + + + /** + * Tokenize a prompt given the native tokenizer + * + * @param prompt the prompt to tokenize + * @return an array of integers each representing a token id + */ + public native int[] encode(String prompt); + + /** + * Convert an array of token ids to its string representation + * + * @param tokens an array of tokens + * @return the token ids decoded to a string + */ + public String decode(int[] tokens) { + byte[] bytes = decodeBytes(tokens); + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * Sets a callback for native llama.cpp log messages. + * Per default, log messages are written in JSON to stdout. Note, that in text mode the callback will be also + * invoked with log messages of the GGML backend, while JSON mode can only access request log messages. + * In JSON mode, GGML messages will still be written to stdout. + * To only change the log format but keep logging to stdout, the given callback can be null. + * To disable logging, pass an empty callback, i.e., (level, msg) -> {}. + * + * @param format the log format to use + * @param callback a method to call for log messages + */ + public static native void setLogger(LogFormat format, @Nullable BiConsumer callback); + + @Override + public void close() { + delete(); + } + + // don't overload native methods since the C++ function names get nasty + native int requestCompletion(String params) throws LlamaException; + + native LlamaOutput receiveCompletion(int taskId) throws LlamaException; + + native void cancelCompletion(int taskId); + + native byte[] decodeBytes(int[] tokens); + + private native void loadModel(String... parameters) throws LlamaException; + + private native void delete(); + + native void releaseTask(int taskId); + + private static native byte[] jsonSchemaToGrammarBytes(String schema); + + public static String jsonSchemaToGrammar(String schema) { + return new String(jsonSchemaToGrammarBytes(schema), StandardCharsets.UTF_8); + } + + public List> rerank(boolean reRank, String query, String ... documents) { + LlamaOutput output = rerank(query, documents); + + Map scoredDocumentMap = output.probabilities; + + List> rankedDocuments = new ArrayList<>(); + + if (reRank) { + // Sort in descending order based on Float values + scoredDocumentMap.entrySet() + .stream() + .sorted((a, b) -> Float.compare(b.getValue(), a.getValue())) // Descending order + .forEach(entry -> rankedDocuments.add(new Pair<>(entry.getKey(), entry.getValue()))); } else { - token = sampleTopK(); - } - - if (params.grammar != null) { - LlamaLibrary.llama_grammar_accept_token(ctx, params.grammar.foreign, token); - } - - return new Output(token, candidateData[token].p); - } - - private void samplePenalty() { - int repeat_last_n = params.repeatLastN < 0 ? params.ctx.n_ctx : params.repeatLastN; - int last_n_repeat = Math.min(Math.min(nContext, repeat_last_n), params.ctx.n_ctx); - NativeSize nTokens = new NativeSize(last_n_repeat); - SliceableIntBuffer lastTokens = tokenBuffer.slice(nContext - last_n_repeat, last_n_repeat); - LlamaLibrary.llama_sample_repetition_penalty( - ctx, - candidates, - lastTokens.delegate, - nTokens, - params.repeatPenalty - ); - LlamaLibrary.llama_sample_frequency_and_presence_penalties( - ctx, - candidates, - lastTokens.delegate, - nTokens, - params.frequencyPenalty, - params.presencePenalty - ); - } - - private int sampleGreedy() { - int token = LlamaLibrary.llama_sample_token_greedy(ctx, candidates); - if (params.nProbs > 0) { - LlamaLibrary.llama_sample_softmax(ctx, candidates); - } - return token; - } - - private int sampleMirostatV1() { - LlamaLibrary.llama_sample_temperature(ctx, candidates, params.temperature); - return LlamaLibrary.llama_sample_token_mirostat( - ctx, - candidates, - params.mirostatTau, - params.mirostatEta, - params.mirostatM, - params.mirostatMu - ); - } - - private int sampleMirostatV2() { - LlamaLibrary.llama_sample_temperature(ctx, candidates, params.temperature); - return LlamaLibrary.llama_sample_token_mirostat_v2( - ctx, - candidates, - params.mirostatTau, - params.mirostatEta, - params.mirostatMu - ); - } - - private int sampleTopK() { - NativeSize minKeep = new NativeSize(Math.max(1, params.nProbs)); - LlamaLibrary.llama_sample_top_k( - ctx, - candidates, - params.topK <= 0 ? nVocab : params.topK, - minKeep - ); - LlamaLibrary.llama_sample_tail_free( - ctx, - candidates, - params.tfsZ, - minKeep - ); - LlamaLibrary.llama_sample_typical( - ctx, - candidates, - params.typicalP, - minKeep - ); - LlamaLibrary.llama_sample_top_p(ctx, - candidates, - params.topP, - minKeep - ); - LlamaLibrary.llama_sample_temperature( - ctx, - candidates, - params.temperature - ); - return LlamaLibrary.llama_sample_token(ctx, candidates); - } - - private void addContext(SliceableIntBuffer tokens) { - truncateContext(tokens.capacity()); - System.arraycopy(tokens.delegate.array(), 0, contextBuffer.delegate.array(), nContext, tokens.capacity()); - nContext += tokens.capacity(); - } - - private void truncateContext(int nAdd) { - if (nContext + nAdd > params.ctx.n_ctx) { - int nCtxKeep = params.ctx.n_ctx / 2 - nAdd; - String msg = "truncating context from " + nContext + " to " + nCtxKeep + " tokens (+" + nAdd + " to add)"; - log(LogLevel.INFO, msg); - System.arraycopy(contextBuffer.delegate.array(), nContext - nCtxKeep, contextBuffer.delegate.array(), 0, nCtxKeep); - nPast = 0; - nContext = nCtxKeep; - } - } - - private String decodeToken(int token) { - int size = LlamaLibrary.llama_token_to_piece(ctx, token, tokenPieceBuffer, tokenPieceBuffer.length); - return new String(tokenPieceBuffer, 0, size, StandardCharsets.UTF_8); - } - - private void log(LogLevel level, String message) { - if (logCallback != null) { - logCallback.accept(level, message); - } - } - - private class LlamaIterator implements Iterator { - - private final StringBuilder builder = new StringBuilder(); - private boolean hasNext = true; - private int nRemain = params.nPredict; - - public LlamaIterator(String prompt) { - setup(prompt); - } - - @Override - public boolean hasNext() { - return hasNext; - } - - @Override - public Output next() { - if (!hasNext) { - throw new NoSuchElementException(); - } - evaluate(); - Output result = sample(); - builder.append(result.text); - truncateContext(1); - contextBuffer.put(nContext, result.token); - nContext++; - nRemain--; - checkHasNext(); - return result; - } - - private void checkHasNext() { - hasNext = (params.nPredict < 0 || nRemain > 0) // if there are infinite or more tokens to predict - && contextBuffer.get(nContext - 1) != tokenEos // if the last token was not a stop token - && !isAntiPrompt(); // if the last tokens don't form an anti-prompt - } - - private void setup(String prompt) { - // add a space in front of the first character to match OG llama tokenizer behavior (taken from main.cpp) - if (nContext == 0 && !prompt.startsWith(" ")) { - prompt = " " + prompt; - } - SliceableIntBuffer tokens = tokenize(prompt, true); - addContext(tokens); - } - - private boolean isAntiPrompt() { - for (String antiPrompt : params.antiprompt) { - if (builder.lastIndexOf(antiPrompt) > 0) { - return true; - } - } - return false; - } - } - - /** - * An output generated by the LLM. Provides access to its probability. - */ - public final class Output { - public final int token; - public final float probability; - public final String text; - - private Output(int token, float probability) { - this.token = token; - this.probability = probability; - // directly convert it to text to do it only once - int size = LlamaLibrary.llama_token_to_piece(ctx, token, tokenPieceBuffer, tokenPieceBuffer.length); - text = new String(tokenPieceBuffer, 0, Math.min(size, tokenPieceBuffer.length), StandardCharsets.UTF_8); - } - - @Override - public String toString() { - return text; - } - - /** - * Returns the type of this token. - * - * @return the token type - */ - public TokenType getType() { - int code = LlamaLibrary.llama_token_get_type(ctx, token); - return TokenType.fromCode(code); - } - - /** - * Returns the score of this token - * - * @return the float score - */ - public float getScore() { - return LlamaLibrary.llama_token_get_score(ctx, token); - } - } + // Copy without sorting + scoredDocumentMap.forEach((key, value) -> rankedDocuments.add(new Pair<>(key, value))); + } + + return rankedDocuments; + } + + public native LlamaOutput rerank(String query, String... documents); + + public String applyTemplate(InferenceParameters parameters) { + return applyTemplate(parameters.toString()); + } + public native String applyTemplate(String parametersJson); } diff --git a/src/main/java/de/kherud/llama/LlamaOutput.java b/src/main/java/de/kherud/llama/LlamaOutput.java new file mode 100644 index 000000000..365b335e0 --- /dev/null +++ b/src/main/java/de/kherud/llama/LlamaOutput.java @@ -0,0 +1,39 @@ +package de.kherud.llama; + +import org.jetbrains.annotations.NotNull; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * An output of the LLM providing access to the generated text and the associated probabilities. You have to configure + * {@link InferenceParameters#setNProbs(int)} in order for probabilities to be returned. + */ +public final class LlamaOutput { + + /** + * The last bit of generated text that is representable as text (i.e., cannot be individual utf-8 multibyte code + * points). + */ + @NotNull + public final String text; + + /** + * Note, that you have to configure {@link InferenceParameters#setNProbs(int)} in order for probabilities to be returned. + */ + @NotNull + public final Map probabilities; + + final boolean stop; + + LlamaOutput(byte[] generated, @NotNull Map probabilities, boolean stop) { + this.text = new String(generated, StandardCharsets.UTF_8); + this.probabilities = probabilities; + this.stop = stop; + } + + @Override + public String toString() { + return text; + } +} diff --git a/src/main/java/de/kherud/llama/LogLevel.java b/src/main/java/de/kherud/llama/LogLevel.java index 903780f7b..b55c08986 100644 --- a/src/main/java/de/kherud/llama/LogLevel.java +++ b/src/main/java/de/kherud/llama/LogLevel.java @@ -1,50 +1,13 @@ package de.kherud.llama; -import de.kherud.llama.foreign.LlamaLibrary; - /** * This enum represents the native log levels of llama.cpp. */ public enum LogLevel { - DEBUG(-1), - INFO(LlamaLibrary.llama_log_level.LLAMA_LOG_LEVEL_INFO), - WARN(LlamaLibrary.llama_log_level.LLAMA_LOG_LEVEL_WARN), - ERROR(LlamaLibrary.llama_log_level.LLAMA_LOG_LEVEL_ERROR); - - private final int code; - - LogLevel(int code) { - this.code = code; - } - - /** - * Returns the native log level code of this option - * - * @return the native code - */ - int getCode() { - return code; - } - - /** - * Returns a Java enum option given a native log level code. - * For unknown native codes the level {@link #DEBUG} is returned instead of throwing. - * - * @param code the native log level code - * @return the Java level representation - */ - static LogLevel fromCode(int code) { - switch (code) { - case LlamaLibrary.llama_log_level.LLAMA_LOG_LEVEL_INFO: - return INFO; - case LlamaLibrary.llama_log_level.LLAMA_LOG_LEVEL_WARN: - return WARN; - case LlamaLibrary.llama_log_level.LLAMA_LOG_LEVEL_ERROR: - return ERROR; - default: - return DEBUG; - } - } + DEBUG, + INFO, + WARN, + ERROR } diff --git a/src/main/java/de/kherud/llama/ModelParameters.java b/src/main/java/de/kherud/llama/ModelParameters.java new file mode 100644 index 000000000..7999295d8 --- /dev/null +++ b/src/main/java/de/kherud/llama/ModelParameters.java @@ -0,0 +1,964 @@ +package de.kherud.llama; + +import de.kherud.llama.args.*; + +/*** + * Parameters used for initializing a {@link LlamaModel}. + */ +@SuppressWarnings("unused") +public final class ModelParameters extends CliParameters { + + /** + * Set the number of threads to use during generation (default: -1). + */ + public ModelParameters setThreads(int nThreads) { + parameters.put("--threads", String.valueOf(nThreads)); + return this; + } + + /** + * Set the number of threads to use during batch and prompt processing (default: same as --threads). + */ + public ModelParameters setThreadsBatch(int nThreads) { + parameters.put("--threads-batch", String.valueOf(nThreads)); + return this; + } + + /** + * Set the CPU affinity mask: arbitrarily long hex. Complements cpu-range (default: ""). + */ + public ModelParameters setCpuMask(String mask) { + parameters.put("--cpu-mask", mask); + return this; + } + + /** + * Set the range of CPUs for affinity. Complements --cpu-mask. + */ + public ModelParameters setCpuRange(String range) { + parameters.put("--cpu-range", range); + return this; + } + + /** + * Use strict CPU placement (default: 0). + */ + public ModelParameters setCpuStrict(int strictCpu) { + parameters.put("--cpu-strict", String.valueOf(strictCpu)); + return this; + } + + /** + * Set process/thread priority: 0-normal, 1-medium, 2-high, 3-realtime (default: 0). + */ + public ModelParameters setPriority(int priority) { + if (priority < 0 || priority > 3) { + throw new IllegalArgumentException("Invalid value for priority"); + } + parameters.put("--prio", String.valueOf(priority)); + return this; + } + + /** + * Set the polling level to wait for work (0 - no polling, default: 0). + */ + public ModelParameters setPoll(int poll) { + parameters.put("--poll", String.valueOf(poll)); + return this; + } + + /** + * Set the CPU affinity mask for batch processing: arbitrarily long hex. Complements cpu-range-batch (default: same as --cpu-mask). + */ + public ModelParameters setCpuMaskBatch(String mask) { + parameters.put("--cpu-mask-batch", mask); + return this; + } + + /** + * Set the ranges of CPUs for batch affinity. Complements --cpu-mask-batch. + */ + public ModelParameters setCpuRangeBatch(String range) { + parameters.put("--cpu-range-batch", range); + return this; + } + + /** + * Use strict CPU placement for batch processing (default: same as --cpu-strict). + */ + public ModelParameters setCpuStrictBatch(int strictCpuBatch) { + parameters.put("--cpu-strict-batch", String.valueOf(strictCpuBatch)); + return this; + } + + /** + * Set process/thread priority for batch processing: 0-normal, 1-medium, 2-high, 3-realtime (default: 0). + */ + public ModelParameters setPriorityBatch(int priorityBatch) { + if (priorityBatch < 0 || priorityBatch > 3) { + throw new IllegalArgumentException("Invalid value for priority batch"); + } + parameters.put("--prio-batch", String.valueOf(priorityBatch)); + return this; + } + + /** + * Set the polling level for batch processing (default: same as --poll). + */ + public ModelParameters setPollBatch(int pollBatch) { + parameters.put("--poll-batch", String.valueOf(pollBatch)); + return this; + } + + /** + * Set the size of the prompt context (default: 0, 0 = loaded from model). + */ + public ModelParameters setCtxSize(int ctxSize) { + parameters.put("--ctx-size", String.valueOf(ctxSize)); + return this; + } + + /** + * Set the number of tokens to predict (default: -1 = infinity, -2 = until context filled). + */ + public ModelParameters setPredict(int nPredict) { + parameters.put("--predict", String.valueOf(nPredict)); + return this; + } + + /** + * Set the logical maximum batch size (default: 0). + */ + public ModelParameters setBatchSize(int batchSize) { + parameters.put("--batch-size", String.valueOf(batchSize)); + return this; + } + + /** + * Set the physical maximum batch size (default: 0). + */ + public ModelParameters setUbatchSize(int ubatchSize) { + parameters.put("--ubatch-size", String.valueOf(ubatchSize)); + return this; + } + + /** + * Set the number of tokens to keep from the initial prompt (default: -1 = all). + */ + public ModelParameters setKeep(int keep) { + parameters.put("--keep", String.valueOf(keep)); + return this; + } + + /** + * Disable context shift on infinite text generation (default: enabled). + */ + public ModelParameters disableContextShift() { + parameters.put("--no-context-shift", null); + return this; + } + + /** + * Enable Flash Attention (default: disabled). + */ + public ModelParameters enableFlashAttn() { + parameters.put("--flash-attn", null); + return this; + } + + /** + * Disable internal libllama performance timings (default: false). + */ + public ModelParameters disablePerf() { + parameters.put("--no-perf", null); + return this; + } + + /** + * Process escape sequences (default: true). + */ + public ModelParameters enableEscape() { + parameters.put("--escape", null); + return this; + } + + /** + * Do not process escape sequences (default: false). + */ + public ModelParameters disableEscape() { + parameters.put("--no-escape", null); + return this; + } + + /** + * Enable special tokens output (default: true). + */ + public ModelParameters enableSpecial() { + parameters.put("--special", null); + return this; + } + + /** + * Skip warming up the model with an empty run (default: false). + */ + public ModelParameters skipWarmup() { + parameters.put("--no-warmup", null); + return this; + } + + /** + * Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. + * (default: disabled) + */ + public ModelParameters setSpmInfill() { + parameters.put("--spm-infill", null); + return this; + } + + /** + * Set samplers that will be used for generation in the order, separated by ';' (default: all). + */ + public ModelParameters setSamplers(Sampler... samplers) { + if (samplers.length > 0) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < samplers.length; i++) { + Sampler sampler = samplers[i]; + builder.append(sampler.name().toLowerCase()); + if (i < samplers.length - 1) { + builder.append(";"); + } + } + parameters.put("--samplers", builder.toString()); + } + return this; + } + + /** + * Set RNG seed (default: -1, use random seed). + */ + public ModelParameters setSeed(long seed) { + parameters.put("--seed", String.valueOf(seed)); + return this; + } + + /** + * Ignore end of stream token and continue generating (implies --logit-bias EOS-inf). + */ + public ModelParameters ignoreEos() { + parameters.put("--ignore-eos", null); + return this; + } + + /** + * Set temperature for sampling (default: 0.8). + */ + public ModelParameters setTemp(float temp) { + parameters.put("--temp", String.valueOf(temp)); + return this; + } + + /** + * Set top-k sampling (default: 40, 0 = disabled). + */ + public ModelParameters setTopK(int topK) { + parameters.put("--top-k", String.valueOf(topK)); + return this; + } + + /** + * Set top-p sampling (default: 0.95, 1.0 = disabled). + */ + public ModelParameters setTopP(float topP) { + parameters.put("--top-p", String.valueOf(topP)); + return this; + } + + /** + * Set min-p sampling (default: 0.05, 0.0 = disabled). + */ + public ModelParameters setMinP(float minP) { + parameters.put("--min-p", String.valueOf(minP)); + return this; + } + + /** + * Set xtc probability (default: 0.0, 0.0 = disabled). + */ + public ModelParameters setXtcProbability(float xtcProbability) { + parameters.put("--xtc-probability", String.valueOf(xtcProbability)); + return this; + } + + /** + * Set xtc threshold (default: 0.1, 1.0 = disabled). + */ + public ModelParameters setXtcThreshold(float xtcThreshold) { + parameters.put("--xtc-threshold", String.valueOf(xtcThreshold)); + return this; + } + + /** + * Set locally typical sampling parameter p (default: 1.0, 1.0 = disabled). + */ + public ModelParameters setTypical(float typP) { + parameters.put("--typical", String.valueOf(typP)); + return this; + } + + /** + * Set last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size). + */ + public ModelParameters setRepeatLastN(int repeatLastN) { + if (repeatLastN < -1) { + throw new RuntimeException("Invalid repeat-last-n value"); + } + parameters.put("--repeat-last-n", String.valueOf(repeatLastN)); + return this; + } + + /** + * Set penalize repeat sequence of tokens (default: 1.0, 1.0 = disabled). + */ + public ModelParameters setRepeatPenalty(float repeatPenalty) { + parameters.put("--repeat-penalty", String.valueOf(repeatPenalty)); + return this; + } + + /** + * Set repeat alpha presence penalty (default: 0.0, 0.0 = disabled). + */ + public ModelParameters setPresencePenalty(float presencePenalty) { + parameters.put("--presence-penalty", String.valueOf(presencePenalty)); + return this; + } + + /** + * Set repeat alpha frequency penalty (default: 0.0, 0.0 = disabled). + */ + public ModelParameters setFrequencyPenalty(float frequencyPenalty) { + parameters.put("--frequency-penalty", String.valueOf(frequencyPenalty)); + return this; + } + + /** + * Set DRY sampling multiplier (default: 0.0, 0.0 = disabled). + */ + public ModelParameters setDryMultiplier(float dryMultiplier) { + parameters.put("--dry-multiplier", String.valueOf(dryMultiplier)); + return this; + } + + /** + * Set DRY sampling base value (default: 1.75). + */ + public ModelParameters setDryBase(float dryBase) { + parameters.put("--dry-base", String.valueOf(dryBase)); + return this; + } + + /** + * Set allowed length for DRY sampling (default: 2). + */ + public ModelParameters setDryAllowedLength(int dryAllowedLength) { + parameters.put("--dry-allowed-length", String.valueOf(dryAllowedLength)); + return this; + } + + /** + * Set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size). + */ + public ModelParameters setDryPenaltyLastN(int dryPenaltyLastN) { + if (dryPenaltyLastN < -1) { + throw new RuntimeException("Invalid dry-penalty-last-n value"); + } + parameters.put("--dry-penalty-last-n", String.valueOf(dryPenaltyLastN)); + return this; + } + + /** + * Add sequence breaker for DRY sampling, clearing out default breakers (default: none). + */ + public ModelParameters setDrySequenceBreaker(String drySequenceBreaker) { + parameters.put("--dry-sequence-breaker", drySequenceBreaker); + return this; + } + + /** + * Set dynamic temperature range (default: 0.0, 0.0 = disabled). + */ + public ModelParameters setDynatempRange(float dynatempRange) { + parameters.put("--dynatemp-range", String.valueOf(dynatempRange)); + return this; + } + + /** + * Set dynamic temperature exponent (default: 1.0). + */ + public ModelParameters setDynatempExponent(float dynatempExponent) { + parameters.put("--dynatemp-exp", String.valueOf(dynatempExponent)); + return this; + } + + /** + * Use Mirostat sampling (default: PLACEHOLDER, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0). + */ + public ModelParameters setMirostat(MiroStat mirostat) { + parameters.put("--mirostat", String.valueOf(mirostat.ordinal())); + return this; + } + + /** + * Set Mirostat learning rate, parameter eta (default: 0.1). + */ + public ModelParameters setMirostatLR(float mirostatLR) { + parameters.put("--mirostat-lr", String.valueOf(mirostatLR)); + return this; + } + + /** + * Set Mirostat target entropy, parameter tau (default: 5.0). + */ + public ModelParameters setMirostatEnt(float mirostatEnt) { + parameters.put("--mirostat-ent", String.valueOf(mirostatEnt)); + return this; + } + + /** + * Modify the likelihood of token appearing in the completion. + */ + public ModelParameters setLogitBias(String tokenIdAndBias) { + parameters.put("--logit-bias", tokenIdAndBias); + return this; + } + + /** + * Set BNF-like grammar to constrain generations (default: empty). + */ + public ModelParameters setGrammar(String grammar) { + parameters.put("--grammar", grammar); + return this; + } + + /** + * Specify the file to read grammar from. + */ + public ModelParameters setGrammarFile(String fileName) { + parameters.put("--grammar-file", fileName); + return this; + } + + /** + * Specify the JSON schema to constrain generations (default: empty). + */ + public ModelParameters setJsonSchema(String schema) { + parameters.put("--json-schema", schema); + return this; + } + + /** + * Set pooling type for embeddings (default: model default if unspecified). + */ + public ModelParameters setPoolingType(PoolingType type) { + parameters.put("--pooling", type.getArgValue()); + return this; + } + + /** + * Set RoPE frequency scaling method (default: linear unless specified by the model). + */ + public ModelParameters setRopeScaling(RopeScalingType type) { + parameters.put("--rope-scaling", type.getArgValue()); + return this; + } + + /** + * Set RoPE context scaling factor, expands context by a factor of N. + */ + public ModelParameters setRopeScale(float ropeScale) { + parameters.put("--rope-scale", String.valueOf(ropeScale)); + return this; + } + + /** + * Set RoPE base frequency, used by NTK-aware scaling (default: loaded from model). + */ + public ModelParameters setRopeFreqBase(float ropeFreqBase) { + parameters.put("--rope-freq-base", String.valueOf(ropeFreqBase)); + return this; + } + + /** + * Set RoPE frequency scaling factor, expands context by a factor of 1/N. + */ + public ModelParameters setRopeFreqScale(float ropeFreqScale) { + parameters.put("--rope-freq-scale", String.valueOf(ropeFreqScale)); + return this; + } + + /** + * Set YaRN: original context size of model (default: model training context size). + */ + public ModelParameters setYarnOrigCtx(int yarnOrigCtx) { + parameters.put("--yarn-orig-ctx", String.valueOf(yarnOrigCtx)); + return this; + } + + /** + * Set YaRN: extrapolation mix factor (default: 0.0 = full interpolation). + */ + public ModelParameters setYarnExtFactor(float yarnExtFactor) { + parameters.put("--yarn-ext-factor", String.valueOf(yarnExtFactor)); + return this; + } + + /** + * Set YaRN: scale sqrt(t) or attention magnitude (default: 1.0). + */ + public ModelParameters setYarnAttnFactor(float yarnAttnFactor) { + parameters.put("--yarn-attn-factor", String.valueOf(yarnAttnFactor)); + return this; + } + + /** + * Set YaRN: high correction dim or alpha (default: 1.0). + */ + public ModelParameters setYarnBetaSlow(float yarnBetaSlow) { + parameters.put("--yarn-beta-slow", String.valueOf(yarnBetaSlow)); + return this; + } + + /** + * Set YaRN: low correction dim or beta (default: 32.0). + */ + public ModelParameters setYarnBetaFast(float yarnBetaFast) { + parameters.put("--yarn-beta-fast", String.valueOf(yarnBetaFast)); + return this; + } + + /** + * Set group-attention factor (default: 1). + */ + public ModelParameters setGrpAttnN(int grpAttnN) { + parameters.put("--grp-attn-n", String.valueOf(grpAttnN)); + return this; + } + + /** + * Set group-attention width (default: 512). + */ + public ModelParameters setGrpAttnW(int grpAttnW) { + parameters.put("--grp-attn-w", String.valueOf(grpAttnW)); + return this; + } + + /** + * Enable verbose printing of the KV cache. + */ + public ModelParameters enableDumpKvCache() { + parameters.put("--dump-kv-cache", null); + return this; + } + + /** + * Disable KV offload. + */ + public ModelParameters disableKvOffload() { + parameters.put("--no-kv-offload", null); + return this; + } + + /** + * Set KV cache data type for K (allowed values: F16). + */ + public ModelParameters setCacheTypeK(CacheType type) { + parameters.put("--cache-type-k", type.name().toLowerCase()); + return this; + } + + /** + * Set KV cache data type for V (allowed values: F16). + */ + public ModelParameters setCacheTypeV(CacheType type) { + parameters.put("--cache-type-v", type.name().toLowerCase()); + return this; + } + + /** + * Set KV cache defragmentation threshold (default: 0.1, < 0 - disabled). + */ + public ModelParameters setDefragThold(float defragThold) { + parameters.put("--defrag-thold", String.valueOf(defragThold)); + return this; + } + + /** + * Set the number of parallel sequences to decode (default: 1). + */ + public ModelParameters setParallel(int nParallel) { + parameters.put("--parallel", String.valueOf(nParallel)); + return this; + } + + /** + * Enable continuous batching (a.k.a dynamic batching) (default: disabled). + */ + public ModelParameters enableContBatching() { + parameters.put("--cont-batching", null); + return this; + } + + /** + * Disable continuous batching. + */ + public ModelParameters disableContBatching() { + parameters.put("--no-cont-batching", null); + return this; + } + + /** + * Force system to keep model in RAM rather than swapping or compressing. + */ + public ModelParameters enableMlock() { + parameters.put("--mlock", null); + return this; + } + + /** + * Do not memory-map model (slower load but may reduce pageouts if not using mlock). + */ + public ModelParameters disableMmap() { + parameters.put("--no-mmap", null); + return this; + } + + /** + * Set NUMA optimization type for system. + */ + public ModelParameters setNuma(NumaStrategy numaStrategy) { + parameters.put("--numa", numaStrategy.name().toLowerCase()); + return this; + } + + /** + * Set comma-separated list of devices to use for offloading <dev1,dev2,..> (none = don't offload). + */ + public ModelParameters setDevices(String devices) { + parameters.put("--device", devices); + return this; + } + + /** + * Set the number of layers to store in VRAM. + */ + public ModelParameters setGpuLayers(int gpuLayers) { + parameters.put("--gpu-layers", String.valueOf(gpuLayers)); + return this; + } + + /** + * Set how to split the model across multiple GPUs (none, layer, row). + */ + public ModelParameters setSplitMode(GpuSplitMode splitMode) { + parameters.put("--split-mode", splitMode.name().toLowerCase()); + return this; + } + + /** + * Set fraction of the model to offload to each GPU, comma-separated list of proportions N0,N1,N2,.... + */ + public ModelParameters setTensorSplit(String tensorSplit) { + parameters.put("--tensor-split", tensorSplit); + return this; + } + + /** + * Set the GPU to use for the model (with split-mode = none), or for intermediate results and KV (with split-mode = row). + */ + public ModelParameters setMainGpu(int mainGpu) { + parameters.put("--main-gpu", String.valueOf(mainGpu)); + return this; + } + + /** + * Enable checking model tensor data for invalid values. + */ + public ModelParameters enableCheckTensors() { + parameters.put("--check-tensors", null); + return this; + } + + /** + * Override model metadata by key. This option can be specified multiple times. + */ + public ModelParameters setOverrideKv(String keyValue) { + parameters.put("--override-kv", keyValue); + return this; + } + + /** + * Add a LoRA adapter (can be repeated to use multiple adapters). + */ + public ModelParameters addLoraAdapter(String fname) { + parameters.put("--lora", fname); + return this; + } + + /** + * Add a LoRA adapter with user-defined scaling (can be repeated to use multiple adapters). + */ + public ModelParameters addLoraScaledAdapter(String fname, float scale) { + parameters.put("--lora-scaled", fname + "," + scale); + return this; + } + + /** + * Add a control vector (this argument can be repeated to add multiple control vectors). + */ + public ModelParameters addControlVector(String fname) { + parameters.put("--control-vector", fname); + return this; + } + + /** + * Add a control vector with user-defined scaling (can be repeated to add multiple scaled control vectors). + */ + public ModelParameters addControlVectorScaled(String fname, float scale) { + parameters.put("--control-vector-scaled", fname + "," + scale); + return this; + } + + /** + * Set the layer range to apply the control vector(s) to (start and end inclusive). + */ + public ModelParameters setControlVectorLayerRange(int start, int end) { + parameters.put("--control-vector-layer-range", start + "," + end); + return this; + } + + /** + * Set the model path from which to load the base model. + */ + public ModelParameters setModel(String model) { + parameters.put("--model", model); + return this; + } + + /** + * Set the model download URL (default: unused). + */ + public ModelParameters setModelUrl(String modelUrl) { + parameters.put("--model-url", modelUrl); + return this; + } + + /** + * Set the Hugging Face model repository (default: unused). + */ + public ModelParameters setHfRepo(String hfRepo) { + parameters.put("--hf-repo", hfRepo); + return this; + } + + /** + * Set the Hugging Face model file (default: unused). + */ + public ModelParameters setHfFile(String hfFile) { + parameters.put("--hf-file", hfFile); + return this; + } + + /** + * Set the Hugging Face model repository for the vocoder model (default: unused). + */ + public ModelParameters setHfRepoV(String hfRepoV) { + parameters.put("--hf-repo-v", hfRepoV); + return this; + } + + /** + * Set the Hugging Face model file for the vocoder model (default: unused). + */ + public ModelParameters setHfFileV(String hfFileV) { + parameters.put("--hf-file-v", hfFileV); + return this; + } + + /** + * Set the Hugging Face access token (default: value from HF_TOKEN environment variable). + */ + public ModelParameters setHfToken(String hfToken) { + parameters.put("--hf-token", hfToken); + return this; + } + + /** + * Enable embedding use case; use only with dedicated embedding models. + */ + public ModelParameters enableEmbedding() { + parameters.put("--embedding", null); + return this; + } + + /** + * Enable reranking endpoint on server. + */ + public ModelParameters enableReranking() { + parameters.put("--reranking", null); + return this; + } + + /** + * Set minimum chunk size to attempt reusing from the cache via KV shifting. + */ + public ModelParameters setCacheReuse(int cacheReuse) { + parameters.put("--cache-reuse", String.valueOf(cacheReuse)); + return this; + } + + /** + * Set the path to save the slot kv cache. + */ + public ModelParameters setSlotSavePath(String slotSavePath) { + parameters.put("--slot-save-path", slotSavePath); + return this; + } + + /** + * Set custom jinja chat template. + */ + public ModelParameters setChatTemplate(String chatTemplate) { + parameters.put("--chat-template", chatTemplate); + return this; + } + + /** + * Set how much the prompt of a request must match the prompt of a slot in order to use that slot. + */ + public ModelParameters setSlotPromptSimilarity(float similarity) { + parameters.put("--slot-prompt-similarity", String.valueOf(similarity)); + return this; + } + + /** + * Load LoRA adapters without applying them (apply later via POST /lora-adapters). + */ + public ModelParameters setLoraInitWithoutApply() { + parameters.put("--lora-init-without-apply", null); + return this; + } + + /** + * Disable logging. + */ + public ModelParameters disableLog() { + parameters.put("--log-disable", null); + return this; + } + + /** + * Set the log file path. + */ + public ModelParameters setLogFile(String logFile) { + parameters.put("--log-file", logFile); + return this; + } + + /** + * Set verbosity level to infinity (log all messages, useful for debugging). + */ + public ModelParameters setVerbose() { + parameters.put("--verbose", null); + return this; + } + + /** + * Set the verbosity threshold (messages with a higher verbosity will be ignored). + */ + public ModelParameters setLogVerbosity(int verbosity) { + parameters.put("--log-verbosity", String.valueOf(verbosity)); + return this; + } + + /** + * Enable prefix in log messages. + */ + public ModelParameters enableLogPrefix() { + parameters.put("--log-prefix", null); + return this; + } + + /** + * Enable timestamps in log messages. + */ + public ModelParameters enableLogTimestamps() { + parameters.put("--log-timestamps", null); + return this; + } + + /** + * Set the number of tokens to draft for speculative decoding. + */ + public ModelParameters setDraftMax(int draftMax) { + parameters.put("--draft-max", String.valueOf(draftMax)); + return this; + } + + /** + * Set the minimum number of draft tokens to use for speculative decoding. + */ + public ModelParameters setDraftMin(int draftMin) { + parameters.put("--draft-min", String.valueOf(draftMin)); + return this; + } + + /** + * Set the minimum speculative decoding probability for greedy decoding. + */ + public ModelParameters setDraftPMin(float draftPMin) { + parameters.put("--draft-p-min", String.valueOf(draftPMin)); + return this; + } + + /** + * Set the size of the prompt context for the draft model. + */ + public ModelParameters setCtxSizeDraft(int ctxSizeDraft) { + parameters.put("--ctx-size-draft", String.valueOf(ctxSizeDraft)); + return this; + } + + /** + * Set the comma-separated list of devices to use for offloading the draft model. + */ + public ModelParameters setDeviceDraft(String deviceDraft) { + parameters.put("--device-draft", deviceDraft); + return this; + } + + /** + * Set the number of layers to store in VRAM for the draft model. + */ + public ModelParameters setGpuLayersDraft(int gpuLayersDraft) { + parameters.put("--gpu-layers-draft", String.valueOf(gpuLayersDraft)); + return this; + } + + /** + * Set the draft model for speculative decoding. + */ + public ModelParameters setModelDraft(String modelDraft) { + parameters.put("--model-draft", modelDraft); + return this; + } + + /** + * Enable jinja for templating + */ + public ModelParameters enableJinja() { + parameters.put("--jinja", null); + return this; + } + +} + + diff --git a/src/main/java/de/kherud/llama/OSInfo.java b/src/main/java/de/kherud/llama/OSInfo.java new file mode 100644 index 000000000..9354ec2ff --- /dev/null +++ b/src/main/java/de/kherud/llama/OSInfo.java @@ -0,0 +1,286 @@ +/*-------------------------------------------------------------------------- + * Copyright 2008 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ + +package de.kherud.llama; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Locale; +import java.util.stream.Stream; + +/** + * Provides OS name and architecture name. + * + * @author leo + */ +@SuppressWarnings("UseOfSystemOutOrSystemErr") +class OSInfo { + public static final String X86 = "x86"; + public static final String X64 = "x64"; + public static final String X86_64 = "x86_64"; + public static final String IA64_32 = "ia64_32"; + public static final String IA64 = "ia64"; + public static final String PPC = "ppc"; + public static final String PPC64 = "ppc64"; + private static final ProcessRunner processRunner = new ProcessRunner(); + private static final HashMap archMapping = new HashMap<>(); + + static { + // x86 mappings + archMapping.put(X86, X86); + archMapping.put("i386", X86); + archMapping.put("i486", X86); + archMapping.put("i586", X86); + archMapping.put("i686", X86); + archMapping.put("pentium", X86); + + // x86_64 mappings + archMapping.put(X86_64, X86_64); + archMapping.put("amd64", X86_64); + archMapping.put("em64t", X86_64); + archMapping.put("universal", X86_64); // Needed for openjdk7 in Mac + + // Itanium 64-bit mappings + archMapping.put(IA64, IA64); + archMapping.put("ia64w", IA64); + + // Itanium 32-bit mappings, usually an HP-UX construct + archMapping.put(IA64_32, IA64_32); + archMapping.put("ia64n", IA64_32); + + // PowerPC mappings + archMapping.put(PPC, PPC); + archMapping.put("power", PPC); + archMapping.put("powerpc", PPC); + archMapping.put("power_pc", PPC); + archMapping.put("power_rs", PPC); + + // TODO: PowerPC 64bit mappings + archMapping.put(PPC64, PPC64); + archMapping.put("power64", PPC64); + archMapping.put("powerpc64", PPC64); + archMapping.put("power_pc64", PPC64); + archMapping.put("power_rs64", PPC64); + archMapping.put("ppc64el", PPC64); + archMapping.put("ppc64le", PPC64); + + // TODO: Adding X64 support + archMapping.put(X64, X64); + } + + public static void main(String[] args) { + if (args.length >= 1) { + if ("--os".equals(args[0])) { + System.out.print(getOSName()); + return; + } + else if ("--arch".equals(args[0])) { + System.out.print(getArchName()); + return; + } + } + + System.out.print(getNativeLibFolderPathForCurrentOS()); + } + + static String getNativeLibFolderPathForCurrentOS() { + return getOSName() + "/" + getArchName(); + } + + static String getOSName() { + return translateOSNameToFolderName(System.getProperty("os.name")); + } + + static boolean isAndroid() { + return isAndroidRuntime() || isAndroidTermux(); + } + + static boolean isAndroidRuntime() { + return System.getProperty("java.runtime.name", "").toLowerCase().contains("android"); + } + + static boolean isAndroidTermux() { + try { + return processRunner.runAndWaitFor("uname -o").toLowerCase().contains("android"); + } + catch (Exception ignored) { + return false; + } + } + + static boolean isMusl() { + Path mapFilesDir = Paths.get("/proc/self/map_files"); + try (Stream dirStream = Files.list(mapFilesDir)) { + return dirStream + .map( + path -> { + try { + return path.toRealPath().toString(); + } + catch (IOException e) { + return ""; + } + }) + .anyMatch(s -> s.toLowerCase().contains("musl")); + } + catch (Exception ignored) { + // fall back to checking for alpine linux in the event we're using an older kernel which + // may not fail the above check + return isAlpineLinux(); + } + } + + static boolean isAlpineLinux() { + try (Stream osLines = Files.lines(Paths.get("/etc/os-release"))) { + return osLines.anyMatch(l -> l.startsWith("ID") && l.contains("alpine")); + } + catch (Exception ignored2) { + } + return false; + } + + static String getHardwareName() { + try { + return processRunner.runAndWaitFor("uname -m"); + } + catch (Throwable e) { + System.err.println("Error while running uname -m: " + e.getMessage()); + return "unknown"; + } + } + + static String resolveArmArchType() { + if (System.getProperty("os.name").contains("Linux")) { + String armType = getHardwareName(); + // armType (uname -m) can be armv5t, armv5te, armv5tej, armv5tejl, armv6, armv7, armv7l, + // aarch64, i686 + + // for Android, we fold everything that is not aarch64 into arm + if (isAndroid()) { + if (armType.startsWith("aarch64")) { + // Use arm64 + return "aarch64"; + } + else { + return "arm"; + } + } + + if (armType.startsWith("armv6")) { + // Raspberry PI + return "armv6"; + } + else if (armType.startsWith("armv7")) { + // Generic + return "armv7"; + } + else if (armType.startsWith("armv5")) { + // Use armv5, soft-float ABI + return "arm"; + } + else if (armType.startsWith("aarch64")) { + // Use arm64 + return "aarch64"; + } + + // Java 1.8 introduces a system property to determine armel or armhf + // https://bugs.openjdk.org/browse/JDK-8005545 + String abi = System.getProperty("sun.arch.abi"); + if (abi != null && abi.startsWith("gnueabihf")) { + return "armv7"; + } + + // For java7, we still need to run some shell commands to determine ABI of JVM + String javaHome = System.getProperty("java.home"); + try { + // determine if first JVM found uses ARM hard-float ABI + int exitCode = Runtime.getRuntime().exec("which readelf").waitFor(); + if (exitCode == 0) { + String[] cmdarray = { + "/bin/sh", + "-c", + "find '" + + javaHome + + "' -name 'libjvm.so' | head -1 | xargs readelf -A | " + + "grep 'Tag_ABI_VFP_args: VFP registers'" + }; + exitCode = Runtime.getRuntime().exec(cmdarray).waitFor(); + if (exitCode == 0) { + return "armv7"; + } + } + else { + System.err.println( + "WARNING! readelf not found. Cannot check if running on an armhf system, armel architecture will be presumed."); + } + } + catch (IOException | InterruptedException e) { + // ignored: fall back to "arm" arch (soft-float ABI) + } + } + // Use armv5, soft-float ABI + return "arm"; + } + + static String getArchName() { + String override = System.getProperty("de.kherud.llama.osinfo.architecture"); + if (override != null) { + return override; + } + + String osArch = System.getProperty("os.arch"); + + if (osArch.startsWith("arm")) { + osArch = resolveArmArchType(); + } + else { + String lc = osArch.toLowerCase(Locale.US); + if (archMapping.containsKey(lc)) return archMapping.get(lc); + } + return translateArchNameToFolderName(osArch); + } + + static String translateOSNameToFolderName(String osName) { + if (osName.contains("Windows")) { + return "Windows"; + } + else if (osName.contains("Mac") || osName.contains("Darwin")) { + return "Mac"; + } + else if (osName.contains("AIX")) { + return "AIX"; + } + else if (isMusl()) { + return "Linux-Musl"; + } + else if (isAndroid()) { + return "Linux-Android"; + } + else if (osName.contains("Linux")) { + return "Linux"; + } + else { + return osName.replaceAll("\\W", ""); + } + } + + static String translateArchNameToFolderName(String archName) { + return archName.replaceAll("\\W", ""); + } +} diff --git a/src/main/java/de/kherud/llama/Pair.java b/src/main/java/de/kherud/llama/Pair.java new file mode 100644 index 000000000..48ac648bb --- /dev/null +++ b/src/main/java/de/kherud/llama/Pair.java @@ -0,0 +1,48 @@ +package de.kherud.llama; + +import java.util.Objects; + +public class Pair { + + private final K key; + private final V value; + + public Pair(K key, V value) { + this.key = key; + this.value = value; + } + + public K getKey() { + return key; + } + + public V getValue() { + return value; + } + + @Override + public int hashCode() { + return Objects.hash(key, value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Pair other = (Pair) obj; + return Objects.equals(key, other.key) && Objects.equals(value, other.value); + } + + @Override + public String toString() { + return "Pair [key=" + key + ", value=" + value + "]"; + } + + + + +} diff --git a/src/main/java/de/kherud/llama/Parameters.java b/src/main/java/de/kherud/llama/Parameters.java deleted file mode 100644 index 598927a24..000000000 --- a/src/main/java/de/kherud/llama/Parameters.java +++ /dev/null @@ -1,746 +0,0 @@ -package de.kherud.llama; - -import java.io.File; -import java.io.IOException; -import java.nio.FloatBuffer; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -import com.sun.jna.Memory; -import com.sun.jna.Pointer; -import com.sun.jna.ptr.FloatByReference; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import de.kherud.llama.foreign.LlamaLibrary; -import de.kherud.llama.foreign.NativeSize; -import de.kherud.llama.foreign.llama_context_params; - -/** - * Parameters used throughout the lifecycle of a {@link LlamaModel}. - * Note, that they are currently blindly copied from llama.cpp. - * Many won't find a use in this project. - */ -public final class Parameters { - - @NotNull - public final llama_context_params.ByValue ctx; - public final int nThreads; - public final int nPredict; // new tokens to predict - public final int nKeep; // number of tokens to keep from initial prompt - public final int nChunks; // max number of chunks to process (-1 = unlimited) - public final int nProbs; // if greater than 0, output the probabilities of top nProbs tokens. - @NotNull - public final Map logitBias; // logit bias for specific tokens - public final int topK; // <= 0 to use vocab size - public final NativeSize topKMinKeep; - public final float topP; // 1.0 = disabled - public final float tfsZ; // 1.0 = disabled - public final float typicalP; // 1.0 = disabled - public final float temperature; // 1.0 = disabled - public final float repeatPenalty; // 1.0 = disabled - public final int repeatLastN; // last n tokens to penalize (0 = disable penalty, -1 = context size) - public final float frequencyPenalty; // 0.0 = disabled - public final float presencePenalty; // 0.0 = disabled - public final MiroStat mirostat; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0 - public final float mirostatTau; // target entropy - public final float mirostatEta; // learning rate - public final int mirostatM; // ?? - public final FloatBuffer mirostatMu; // ?? - public final boolean beamSearch; - public final int nBeams; - public final String cfgNegativePrompt; // string to help guidance - public final float cfgScale; // How strong is guidance - @Nullable - public final String pathPromptCache; // path to file for saving/loading prompt eval state - @Nullable - public final String inputPrefix; // string to prefix user inputs with - @Nullable - public final String inputSuffix; // string to suffix user inputs with - @Nullable - public final LlamaGrammar grammar; // optional BNF-like grammar to constrain sampling - @NotNull - public final List<@NotNull String> antiprompt; // string upon seeing which more user input is prompted - @Nullable - public final String loraAdapter; // lora adapter path - @Nullable - public final String loraBase; // base model path for the lora adapter - public final boolean hellaswag; // compute HellaSwag score over random tasks from datafile supplied in prompt - @Nullable - public final NativeSize hellaswagTasks; // number of tasks to use when computing the HellaSwag score - public final boolean memoryF16; // use f16 instead of f32 for memory kv - public final boolean randomPrompt; // do not randomize prompt if none provided - public final boolean useColor; // use color to distinguish generations and inputs - public final boolean interactive; // interactive mode - public final boolean promptCacheAll; // save user input and generations to prompt cache - public final boolean promptCacheRo; // open the prompt cache read-only and do not update it - public final boolean interactiveFirst; // wait for user input immediately - public final boolean multilineInput; // reverse the usage of `\` - public final boolean simpleIo; // improves compatibility with subprocesses and limited consoles - public final boolean inputPrefixBos; // prefix BOS to user inputs, preceding inputPrefix - public final boolean instruct; // instruction mode (used for Alpaca models) - public final boolean penalizeNl; // consider newlines as a repeatable token - public final boolean perplexity; // compute perplexity over the prompt - public final boolean memTest; // compute maximum memory usage - public final boolean numa; // attempt optimizations that help on some NUMA systems - public final boolean exportCgraph; // export the computation graph - public final boolean verbosePrompt; // print prompt tokens before generation - - /** - * Private constructor to build immutable parameters object. Called via {@link Builder}. - */ - private Parameters( - llama_context_params.@NotNull ByValue ctx, - int nThreads, - int nPredict, - int nKeep, - int nChunks, - int nProbs, - @NotNull Map logitBias, - int topK, - int topKMinKeep, - float topP, - float tfsZ, - float typicalP, - float temperature, - float repeatPenalty, - int repeatLastN, - float frequencyPenalty, - float presencePenalty, - MiroStat mirostat, - float mirostatTau, - float mirostatEta, - int mirostatM, - float mirostatMu, - boolean beamSearch, - int nBeams, - String cfgNegativePrompt, - float cfgScale, - @Nullable String pathPromptCache, - @Nullable String inputPrefix, - @Nullable String inputSuffix, - @Nullable LlamaGrammar grammar, - @NotNull List antiprompt, - @Nullable String loraAdapter, - @Nullable String loraBase, - boolean hellaswag, - @Nullable NativeSize hellaswagTasks, - boolean memoryF16, - boolean randomPrompt, - boolean useColor, - boolean interactive, - boolean promptCacheAll, - boolean promptCacheRo, - boolean interactiveFirst, - boolean multilineInput, - boolean simpleIo, - boolean inputPrefixBos, - boolean instruct, - boolean penalizeNl, - boolean perplexity, - boolean memTest, - boolean numa, - boolean exportCgraph, - boolean verbosePrompt - ) { - this.ctx = ctx; - this.nThreads = nThreads; - this.nPredict = nPredict; - this.nKeep = nKeep; - this.nChunks = nChunks; - this.nProbs = nProbs; - this.logitBias = logitBias; - this.topK = topK; - this.topKMinKeep = new NativeSize(topKMinKeep); - this.topP = topP; - this.tfsZ = tfsZ; - this.typicalP = typicalP; - this.temperature = temperature; - this.repeatPenalty = repeatPenalty; - this.repeatLastN = repeatLastN; - this.frequencyPenalty = frequencyPenalty; - this.presencePenalty = presencePenalty; - this.mirostat = mirostat; - this.mirostatTau = mirostatTau; - this.mirostatEta = mirostatEta; - this.mirostatM = mirostatM; - this.mirostatMu = FloatBuffer.wrap(new float[]{mirostatMu}); - this.beamSearch = beamSearch; - this.nBeams = nBeams; - this.cfgNegativePrompt = cfgNegativePrompt; - this.cfgScale = cfgScale; - this.pathPromptCache = pathPromptCache; - this.inputPrefix = inputPrefix; - this.inputSuffix = inputSuffix; - this.grammar = grammar; - this.antiprompt = antiprompt; - this.loraAdapter = loraAdapter; - this.loraBase = loraBase; - this.hellaswag = hellaswag; - this.hellaswagTasks = hellaswagTasks; - this.memoryF16 = memoryF16; - this.randomPrompt = randomPrompt; - this.useColor = useColor; - this.interactive = interactive; - this.promptCacheAll = promptCacheAll; - this.promptCacheRo = promptCacheRo; - this.interactiveFirst = interactiveFirst; - this.multilineInput = multilineInput; - this.simpleIo = simpleIo; - this.inputPrefixBos = inputPrefixBos; - this.instruct = instruct; - this.penalizeNl = penalizeNl; - this.perplexity = perplexity; - this.memTest = memTest; - this.numa = numa; - this.exportCgraph = exportCgraph; - this.verbosePrompt = verbosePrompt; - } - - /** - * The builder class used for creating new {@link Parameters} of a {@link LlamaModel}. - */ - public static class Builder { - - private final llama_context_params.ByValue ctxParams = LlamaLibrary.llama_context_default_params(); - - private int nThreads = Runtime.getRuntime().availableProcessors(); - private int nPredict = -1; // new tokens to predict - private int nKeep = 0; // number of tokens to keep from initial prompt - private int nChunks = -1; // max number of chunks to process (-1 = unlimited) - private int nProbs = 0; // if greater than 0, output the probabilities of top nProbs tokens. - - // sampling parameters - private Map logitBias = Collections.emptyMap(); // logit bias for specific tokens - private int topK = 40; // <= 0 to use vocab size - private int topKMinKeep = 1; // <= 0 to use vocab size - private float topP = 0.95f; // 1.0 = disabled - private float tfsZ = 1.00f; // 1.0 = disabled - private float typicalP = 1.00f; // 1.0 = disabled - private float temp = 0.80f; // 1.0 = disabled - private float repeatPenalty = 1.10f; // 1.0 = disabled - private int repeatLastN = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size) - private float frequencyPenalty = 0.00f; // 0.0 = disabled - private float presencePenalty = 0.00f; // 0.0 = disabled - private MiroStat mirostat = MiroStat.Disabled; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0 - private float mirostatTau = 5.00f; // target entropy - private float mirostatEta = 0.10f; // learning rate - private int mirostatM = 100; // ?? - private float mirostatMu = 2f * mirostatTau; // ?? - private boolean beamSearch = false; - private int nBeams = 2; - - // Classifier-Free Guidance - // https://arxiv.org/abs/2306.17806 - private String cfgNegativePrompt; // string to help guidance - private float cfgScale = 1.f; // How strong is guidance - - private String pathPromptCache = null; // path to file for saving/loading prompt eval state - private String inputPrefix = null; // string to prefix user inputs with - private String inputSuffix = null; // string to suffix user inputs with - private LlamaGrammar grammar = null; // optional BNF-like grammar to constrain sampling - private List antiPrompt = Collections.emptyList(); // string upon seeing which more user input is prompted - - private String loraAdapter = null; // lora adapter path - private String loraBase = null; // base model path for the lora adapter - - private boolean hellaswag = false; // compute HellaSwag score over random tasks from datafile supplied in prompt - private NativeSize hellaswagTasks = new NativeSize(400); // number of tasks to use when computing the HellaSwag score - - private boolean memoryF16 = true; // use f16 instead of f32 for memory kv - private boolean randomPrompt = false; // do not randomize prompt if none provided - private boolean useColor = false; // use color to distinguish generations and inputs - private boolean interactive = false; // interactive mode - private boolean promptCacheAll = false; // save user input and generations to prompt cache - private boolean promptCacheRo = false; // open the prompt cache read-only and do not update it - - private boolean interactiveFirst = false; // wait for user input immediately - private boolean multilineInput = false; // reverse the usage of `\` - private boolean simpleIo = false; // improves compatibility with subprocesses and limited consoles - - private boolean inputPrefixBos = false; // prefix BOS to user inputs, preceding inputPrefix - private boolean instruct = false; // instruction mode (used for Alpaca models) - private boolean penalizeNl = true; // consider newlines as a repeatable token - private boolean perplexity = false; // compute perplexity over the prompt - private boolean memTest = false; // compute maximum memory usage - private boolean numa = false; // attempt optimizations that help on some NUMA systems - private boolean exportCgraph = false; // export the computation graph - private boolean verbosePrompt = false; // print prompt tokens before generation - - /** - * Constructs the immutable {@link Parameters} objects with the configured options. - * Note, that all options not configured are expected to have sensible defaults. - * - * @return an immutable parameters object - */ - public Parameters build() { - return new Parameters( - ctxParams, - nThreads, - nPredict, - nKeep, - nChunks, - nProbs, - logitBias, - topK, - topKMinKeep, - topP, - tfsZ, - typicalP, - temp, - repeatPenalty, - repeatLastN, - frequencyPenalty, - presencePenalty, - mirostat, - mirostatTau, - mirostatEta, - mirostatM, - mirostatMu, - beamSearch, - nBeams, - cfgNegativePrompt, - cfgScale, - pathPromptCache, - inputPrefix, - inputSuffix, - grammar, - antiPrompt, - loraAdapter, - loraBase, - hellaswag, - hellaswagTasks, - memoryF16, - randomPrompt, - useColor, - interactive, - promptCacheAll, - promptCacheRo, - interactiveFirst, - multilineInput, - simpleIo, - inputPrefixBos, - instruct, - penalizeNl, - perplexity, - memTest, - numa, - exportCgraph, - verbosePrompt - ); - } - - public Builder setNThreads(int nThreads) { - this.nThreads = nThreads; - return this; - } - - public Builder setNPredict(int nPredict) { - this.nPredict = nPredict; - return this; - } - - public Builder setNKeep(int nKeep) { - this.nKeep = nKeep; - return this; - } - - public Builder setNChunks(int nChunks) { - this.nChunks = nChunks; - return this; - } - - public Builder setNProbs(int nProbs) { - this.nProbs = nProbs; - return this; - } - - public Builder setLogitBias(@NotNull Map logitBias) { - this.logitBias = Collections.unmodifiableMap(logitBias); - return this; - } - - public Builder setTopK(int topK) { - this.topK = topK; - return this; - } - - public Builder setTopKMinKeep(int topKMinKeep) { - this.topKMinKeep = topKMinKeep; - return this; - } - - public Builder setTopP(float topP) { - this.topP = topP; - return this; - } - - public Builder setTfsZ(float tfsZ) { - this.tfsZ = tfsZ; - return this; - } - - public Builder setTypicalP(float typicalP) { - this.typicalP = typicalP; - return this; - } - - public Builder setTemperature(float temp) { - this.temp = temp; - return this; - } - - public Builder setRepeatPenalty(float repeatPenalty) { - this.repeatPenalty = repeatPenalty; - return this; - } - - public Builder setRepeatLastN(int repeatLastN) { - this.repeatLastN = repeatLastN; - return this; - } - - public Builder setFrequencyPenalty(float frequencyPenalty) { - this.frequencyPenalty = frequencyPenalty; - return this; - } - - public Builder setPresencePenalty(float presencePenalty) { - this.presencePenalty = presencePenalty; - return this; - } - - public Builder setMirostat(MiroStat mode) { - this.mirostat = mode; - return this; - } - - public Builder setMirostatTau(float mirostatTau) { - this.mirostatTau = mirostatTau; - return this; - } - - public Builder setMirostatEta(float mirostatEta) { - this.mirostatEta = mirostatEta; - return this; - } - - public Builder setMirostatM(int mirostatM) { - this.mirostatM = mirostatM; - return this; - } - - public Builder setMirostatMu(float mirostatMu) { - this.mirostatMu = mirostatMu; - return this; - } - - public Builder setBeamSearch(boolean beamSearch) { - this.beamSearch = beamSearch; - return this; - } - - public Builder setNBeams(int nBeams) { - this.nBeams = nBeams; - return this; - } - - public Builder setCfgNegativePrompt(@Nullable String cfgNegativePrompt) { - this.cfgNegativePrompt = cfgNegativePrompt; - return this; - } - - public Builder setCfgScale(float cfgScale) { - this.cfgScale = cfgScale; - return this; - } - - public Builder setPathPromptCache(@Nullable String pathPromptCache) { - this.pathPromptCache = pathPromptCache; - return this; - } - - public Builder setInputPrefix(@Nullable String inputPrefix) { - this.inputPrefix = inputPrefix; - return this; - } - - public Builder setInputSuffix(@Nullable String inputSuffix) { - this.inputSuffix = inputSuffix; - return this; - } - - public Builder setGrammar(@NotNull File file) throws IOException { - this.grammar = new LlamaGrammar(file); - return this; - } - - public Builder setGrammar(@NotNull Path path) throws IOException { - this.grammar = new LlamaGrammar(path); - return this; - } - - public Builder setGrammar(@Nullable String grammar) { - this.grammar = grammar == null ? null : new LlamaGrammar(grammar); - return this; - } - - public Builder setAntiPrompt(@NotNull String[] antiprompt) { - this.antiPrompt = Collections.unmodifiableList(Arrays.asList(antiprompt)); - return this; - } - - public Builder setLoraAdapter(@Nullable String loraAdapter) { - this.loraAdapter = loraAdapter; - return this; - } - - public Builder setLoraBase(@Nullable String loraBase) { - this.loraBase = loraBase; - return this; - } - - public Builder setHellaswag(boolean hellaswag) { - this.hellaswag = hellaswag; - return this; - } - - public Builder setHellaswagTasks(long hellaswagTasks) { - this.hellaswagTasks = new NativeSize(hellaswagTasks); - return this; - } - - public Builder setMemoryF16(boolean memoryF16) { - this.memoryF16 = memoryF16; - return this; - } - - public Builder setRandomPrompt(boolean randomPrompt) { - this.randomPrompt = randomPrompt; - return this; - } - - public Builder setUseColor(boolean useColor) { - this.useColor = useColor; - return this; - } - - public Builder setInteractive(boolean interactive) { - this.interactive = interactive; - return this; - } - - public Builder setPromptCacheAll(boolean promptCacheAll) { - this.promptCacheAll = promptCacheAll; - return this; - } - - public Builder setPromptCacheRo(boolean promptCacheRo) { - this.promptCacheRo = promptCacheRo; - return this; - } - - public Builder setInteractiveFirst(boolean interactiveFirst) { - this.interactiveFirst = interactiveFirst; - return this; - } - - public Builder setMultilineInput(boolean multilineInput) { - this.multilineInput = multilineInput; - return this; - } - - public Builder setSimpleIo(boolean simpleIo) { - this.simpleIo = simpleIo; - return this; - } - - public Builder setInputPrefixBos(boolean inputPrefixBos) { - this.inputPrefixBos = inputPrefixBos; - return this; - } - - public Builder setInstruct(boolean instruct) { - this.instruct = instruct; - return this; - } - - public Builder setPenalizeNl(boolean penalizeNl) { - this.penalizeNl = penalizeNl; - return this; - } - - public Builder setPerplexity(boolean perplexity) { - this.perplexity = perplexity; - return this; - } - - public Builder setMemTest(boolean memTest) { - this.memTest = memTest; - return this; - } - - public Builder setNuma(boolean numa) { - this.numa = numa; - return this; - } - - public Builder setExportCGraph(boolean exportCgraph) { - this.exportCgraph = exportCgraph; - return this; - } - - public Builder setVerbosePrompt(boolean verbosePrompt) { - this.verbosePrompt = verbosePrompt; - return this; - } - - /** - * Set a callback that will be used to report progress loading the model with a float value of 0-1. - * - * @param progressCallback the function to call ony any progress - * @return this builder object - */ - public Builder setProgressCallback(@Nullable Consumer progressCallback) { - // Similarly to setting the logger, we don't allow passing any user data to the progress callback, since - // the JVM might move the object around in the memory, thus invalidating any pointers. - if (progressCallback == null) { - ctxParams.setProgress_callback(null); - } else { - ctxParams.setProgress_callback((progress, ctx) -> progressCallback.accept(progress)); - } - return this; - } - - public Builder setSeed(int seed) { - ctxParams.setSeed(seed); - return this; - } - - public Builder setNCtx(int n_ctx) { - ctxParams.setN_ctx(n_ctx); - return this; - } - - public Builder setNBbatch(int n_batch) { - ctxParams.setN_batch(n_batch); - return this; - } - - public Builder setNGpuLayers(int n_gpu_layers) { - ctxParams.setN_gpu_layers(n_gpu_layers); - return this; - } - - public Builder setMainGpu(int main_gpu) { - ctxParams.setMain_gpu(main_gpu); - return this; - } - - public Builder setTensorSplit(float[] tensor_split) { - Memory memory = new Memory(tensor_split.length * 4L); // Allocate native memory, 4 bytes per float - for (int i = 0; i < tensor_split.length; i++) { - memory.setFloat(i * 4L, tensor_split[i]); // Copy the float values to native memory - } - FloatByReference floatByRef = new FloatByReference(); - floatByRef.setPointer(memory); - ctxParams.setTensor_split(floatByRef); - return this; - } - - public Builder setRopeFreqBase(float rope_freq_base) { - ctxParams.setRope_freq_base(rope_freq_base); - return this; - } - - public Builder setRopeFreqScale(float rope_freq_scale) { - ctxParams.setRope_freq_scale(rope_freq_scale); - return this; - } - - public Builder setProgressCallback(LlamaLibrary.llama_progress_callback progress_callback) { - ctxParams.setProgress_callback(progress_callback); - return this; - } - - public Builder setProgressCallbackUserData(Pointer progress_callback_user_data) { - ctxParams.setProgress_callback_user_data(progress_callback_user_data); - return this; - } - - public Builder setLowVram(boolean lowVram) { - ctxParams.setLow_vram((byte) (lowVram ? 1 : 0)); - return this; - } - - public Builder setMulMatQ(boolean mulMatQ) { - ctxParams.setLow_vram((byte) (mulMatQ ? 1 : 0)); - return this; - } - - /** - * use fp16 for KV cache - */ - public Builder setF16Kv(boolean f16_kv) { - ctxParams.setF16_kv(f16_kv ? (byte) 1 : 0); - return this; - } - - /** - * the llama_eval() call computes all logits, not just the last one - */ - public Builder setLogitsAll(boolean logits_all) { - ctxParams.setLogits_all(logits_all ? (byte) 1 : 0); - return this; - } - - /** - * only load the vocabulary, no weights - */ - public Builder setVocabOnly(boolean vocab_only) { - ctxParams.setVocab_only(vocab_only ? (byte) 1 : 0); - return this; - } - - /** - * use mmap if possible - */ - public Builder setUseMmap(boolean use_mmap) { - ctxParams.setUse_mmap(use_mmap ? (byte) 1 : 0); - return this; - } - - /** - * force system to keep model in RAM - */ - public Builder setUseMLock(boolean use_mlock) { - ctxParams.setUse_mlock(use_mlock ? (byte) 1 : 0); - return this; - } - - /** - * embedding mode only - */ - public Builder setEmbedding(boolean embedding) { - ctxParams.setEmbedding(embedding ? (byte) 1 : 0); - return this; - } - } - - public enum MiroStat { - - Disabled(0), - V1(1), - V2(2); - - private final int level; - - MiroStat(int level) { - this.level = level; - } - } -} diff --git a/src/main/java/de/kherud/llama/ProcessRunner.java b/src/main/java/de/kherud/llama/ProcessRunner.java new file mode 100644 index 000000000..24e63498a --- /dev/null +++ b/src/main/java/de/kherud/llama/ProcessRunner.java @@ -0,0 +1,35 @@ +package de.kherud.llama; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.TimeUnit; + +class ProcessRunner { + String runAndWaitFor(String command) throws IOException, InterruptedException { + Process p = Runtime.getRuntime().exec(command); + p.waitFor(); + + return getProcessOutput(p); + } + + String runAndWaitFor(String command, long timeout, TimeUnit unit) + throws IOException, InterruptedException { + Process p = Runtime.getRuntime().exec(command); + p.waitFor(timeout, unit); + + return getProcessOutput(p); + } + + private static String getProcessOutput(Process process) throws IOException { + try (InputStream in = process.getInputStream()) { + int readLen; + ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[32]; + while ((readLen = in.read(buf, 0, buf.length)) >= 0) { + b.write(buf, 0, readLen); + } + return b.toString(); + } + } +} diff --git a/src/main/java/de/kherud/llama/SliceableIntBuffer.java b/src/main/java/de/kherud/llama/SliceableIntBuffer.java deleted file mode 100644 index 2ce769e2a..000000000 --- a/src/main/java/de/kherud/llama/SliceableIntBuffer.java +++ /dev/null @@ -1,76 +0,0 @@ -package de.kherud.llama; - -import java.nio.IntBuffer; - - -/** - * Container that allows slicing an {@link IntBuffer} - * with arbitrary slicing lengths on Java versions older than 13. - * Does not extend IntBuffer because the super constructor - * requires memory segment proxies, and we can't access the delegate's. - * Does not implement Buffer because the {@link java.nio.Buffer#slice(int, int)} - * method is specifically blocked from being implemented or used on older jdk versions. - */ -class SliceableIntBuffer { - final IntBuffer delegate; - - private final int offset; - - private final int capacity; - - SliceableIntBuffer(IntBuffer delegate) { - this.delegate = delegate; - this.capacity = delegate.capacity(); - this.offset = 0; - } - - SliceableIntBuffer(IntBuffer delegate, int offset, int capacity) { - this.delegate = delegate; - this.offset = offset; - this.capacity = capacity; - } - - SliceableIntBuffer slice(int offset, int length) { - // Where the magic happens - // Wrapping is equivalent to the slice operation so long - // as you keep track of your offsets and capacities. - // So, we use this container class to track those offsets and translate - // them to the correct frame of reference. - return new SliceableIntBuffer( - IntBuffer.wrap( - this.delegate.array(), - this.offset + offset, - length - ), - this.offset + offset, - length - ); - - } - - int capacity() { - return capacity; - } - - SliceableIntBuffer put(int index, int i) { - delegate.put(offset + index, i); - return this; - } - - int get(int index) { - return delegate.get(offset + index); - } - - void clear() { - // Clear set the limit and position - // to 0 and capacity respectively, - // but that's not what the buffer was initially - // after the wrap() call, so we manually - // set the limit and position to what they were - // after the wrap call. - delegate.clear(); - delegate.limit(offset + capacity); - delegate.position(offset); - } - -} diff --git a/src/main/java/de/kherud/llama/TokenType.java b/src/main/java/de/kherud/llama/TokenType.java deleted file mode 100644 index eb7c75fe2..000000000 --- a/src/main/java/de/kherud/llama/TokenType.java +++ /dev/null @@ -1,51 +0,0 @@ -package de.kherud.llama; - -import de.kherud.llama.foreign.LlamaLibrary; - -/** - * This enum represents the native token types of llama.cpp. - */ -public enum TokenType { - - UNDEFINED(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_UNDEFINED), - NORMAL(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_NORMAL), - UNKNOWN(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_UNKNOWN), - CONTROL(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_CONTROL), - USER_DEFINED(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_USER_DEFINED), - UNUSED(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_UNUSED), - BYTE(LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_BYTE); - - private final int code; - - TokenType(int code) { - this.code = code; - } - - /** - * Returns a Java enum option given a native token type. - * For unknown native codes {@link #UNKNOWN} is returned instead of throwing. - * - * @param code the native token type - * @return the Java token type representation - */ - static TokenType fromCode(int code) { - switch (code) { - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_UNDEFINED: - return UNDEFINED; - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_NORMAL: - return NORMAL; - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_CONTROL: - return CONTROL; - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_USER_DEFINED: - return USER_DEFINED; - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_UNUSED: - return UNUSED; - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_BYTE: - return BYTE; - case LlamaLibrary.llama_token_type.LLAMA_TOKEN_TYPE_UNKNOWN: - default: - return UNKNOWN; - } - } - -} diff --git a/src/main/java/de/kherud/llama/VocabularyType.java b/src/main/java/de/kherud/llama/VocabularyType.java deleted file mode 100644 index 2bb71644d..000000000 --- a/src/main/java/de/kherud/llama/VocabularyType.java +++ /dev/null @@ -1,34 +0,0 @@ -package de.kherud.llama; - -import de.kherud.llama.foreign.LlamaLibrary; - -/** - * This enum represents the native vocabulary types of llama.cpp. - */ -public enum VocabularyType { - - SENTENCE_PIECE(LlamaLibrary.llama_vocab_type.LLAMA_VOCAB_TYPE_SPM), - BYTE_PAIR(LlamaLibrary.llama_vocab_type.LLAMA_VOCAB_TYPE_BPE); - - private final int code; - - VocabularyType(int code) { - this.code = code; - } - - /** - * Returns a Java enum option given a native vocabulary type. - * For unknown native codes {@link #BYTE_PAIR} is returned instead of throwing. - * - * @param code the native vocabulary type - * @return the Java vocabulary type representation - */ - static VocabularyType fromCode(int code) { - if (code == LlamaLibrary.llama_vocab_type.LLAMA_VOCAB_TYPE_SPM) { - return SENTENCE_PIECE; - } else { - return BYTE_PAIR; - } - } - -} diff --git a/src/main/java/de/kherud/llama/args/CacheType.java b/src/main/java/de/kherud/llama/args/CacheType.java new file mode 100644 index 000000000..8404ed756 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/CacheType.java @@ -0,0 +1,15 @@ +package de.kherud.llama.args; + +public enum CacheType { + + F32, + F16, + BF16, + Q8_0, + Q4_0, + Q4_1, + IQ4_NL, + Q5_0, + Q5_1 + +} diff --git a/src/main/java/de/kherud/llama/args/GpuSplitMode.java b/src/main/java/de/kherud/llama/args/GpuSplitMode.java new file mode 100644 index 000000000..0c0cd9348 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/GpuSplitMode.java @@ -0,0 +1,8 @@ +package de.kherud.llama.args; + +public enum GpuSplitMode { + + NONE, + LAYER, + ROW +} diff --git a/src/main/java/de/kherud/llama/args/LogFormat.java b/src/main/java/de/kherud/llama/args/LogFormat.java new file mode 100644 index 000000000..8a5b46e83 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/LogFormat.java @@ -0,0 +1,11 @@ +package de.kherud.llama.args; + +/** + * The log output format (defaults to JSON for all server-based outputs). + */ +public enum LogFormat { + + JSON, + TEXT + +} diff --git a/src/main/java/de/kherud/llama/args/MiroStat.java b/src/main/java/de/kherud/llama/args/MiroStat.java new file mode 100644 index 000000000..5268d9bc2 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/MiroStat.java @@ -0,0 +1,8 @@ +package de.kherud.llama.args; + +public enum MiroStat { + + DISABLED, + V1, + V2 +} diff --git a/src/main/java/de/kherud/llama/args/NumaStrategy.java b/src/main/java/de/kherud/llama/args/NumaStrategy.java new file mode 100644 index 000000000..fa7a61b0f --- /dev/null +++ b/src/main/java/de/kherud/llama/args/NumaStrategy.java @@ -0,0 +1,8 @@ +package de.kherud.llama.args; + +public enum NumaStrategy { + + DISTRIBUTE, + ISOLATE, + NUMACTL +} diff --git a/src/main/java/de/kherud/llama/args/PoolingType.java b/src/main/java/de/kherud/llama/args/PoolingType.java new file mode 100644 index 000000000..c0379c859 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/PoolingType.java @@ -0,0 +1,21 @@ +package de.kherud.llama.args; + +public enum PoolingType { + + UNSPECIFIED("unspecified"), + NONE("none"), + MEAN("mean"), + CLS("cls"), + LAST("last"), + RANK("rank"); + + private final String argValue; + + PoolingType(String value) { + this.argValue = value; + } + + public String getArgValue() { + return argValue; + } +} \ No newline at end of file diff --git a/src/main/java/de/kherud/llama/args/RopeScalingType.java b/src/main/java/de/kherud/llama/args/RopeScalingType.java new file mode 100644 index 000000000..138d05be3 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/RopeScalingType.java @@ -0,0 +1,21 @@ +package de.kherud.llama.args; + +public enum RopeScalingType { + + UNSPECIFIED("unspecified"), + NONE("none"), + LINEAR("linear"), + YARN2("yarn"), + LONGROPE("longrope"), + MAX_VALUE("maxvalue"); + + private final String argValue; + + RopeScalingType(String value) { + this.argValue = value; + } + + public String getArgValue() { + return argValue; + } +} \ No newline at end of file diff --git a/src/main/java/de/kherud/llama/args/Sampler.java b/src/main/java/de/kherud/llama/args/Sampler.java new file mode 100644 index 000000000..564a2e6f8 --- /dev/null +++ b/src/main/java/de/kherud/llama/args/Sampler.java @@ -0,0 +1,15 @@ +package de.kherud.llama.args; + +public enum Sampler { + + DRY, + TOP_K, + TOP_P, + TYP_P, + MIN_P, + TEMPERATURE, + XTC, + INFILL, + PENALTIES + +} diff --git a/src/main/java/de/kherud/llama/foreign/LlamaLibrary.java b/src/main/java/de/kherud/llama/foreign/LlamaLibrary.java deleted file mode 100644 index 32046c7ca..000000000 --- a/src/main/java/de/kherud/llama/foreign/LlamaLibrary.java +++ /dev/null @@ -1,187 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Callback; -import com.sun.jna.Library; -import com.sun.jna.Native; -import com.sun.jna.NativeLibrary; -import com.sun.jna.Pointer; -import com.sun.jna.PointerType; -import com.sun.jna.ptr.FloatByReference; -import com.sun.jna.ptr.PointerByReference; -import java.nio.ByteBuffer; -import java.nio.FloatBuffer; -import java.nio.IntBuffer; -/** - * JNA Wrapper for library llama
- * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class LlamaLibrary implements Library { - public static final String JNA_LIBRARY_NAME = "llama"; - public static final NativeLibrary JNA_NATIVE_LIB = NativeLibrary.getInstance(LlamaLibrary.JNA_LIBRARY_NAME); - static { - Native.register(LlamaLibrary.class, LlamaLibrary.JNA_NATIVE_LIB); - } - public static interface llama_log_level { - public static final int LLAMA_LOG_LEVEL_ERROR = 2; - public static final int LLAMA_LOG_LEVEL_WARN = 3; - public static final int LLAMA_LOG_LEVEL_INFO = 4; - }; - public static interface llama_vocab_type { - public static final int LLAMA_VOCAB_TYPE_SPM = 0; - public static final int LLAMA_VOCAB_TYPE_BPE = 1; - }; - public static interface llama_token_type { - public static final int LLAMA_TOKEN_TYPE_UNDEFINED = 0; - public static final int LLAMA_TOKEN_TYPE_NORMAL = 1; - public static final int LLAMA_TOKEN_TYPE_UNKNOWN = 2; - public static final int LLAMA_TOKEN_TYPE_CONTROL = 3; - public static final int LLAMA_TOKEN_TYPE_USER_DEFINED = 4; - public static final int LLAMA_TOKEN_TYPE_UNUSED = 5; - public static final int LLAMA_TOKEN_TYPE_BYTE = 6; - }; - public static interface llama_ftype { - public static final int LLAMA_FTYPE_ALL_F32 = 0; - public static final int LLAMA_FTYPE_MOSTLY_F16 = 1; - public static final int LLAMA_FTYPE_MOSTLY_Q4_0 = 2; - public static final int LLAMA_FTYPE_MOSTLY_Q4_1 = 3; - public static final int LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4; - public static final int LLAMA_FTYPE_MOSTLY_Q8_0 = 7; - public static final int LLAMA_FTYPE_MOSTLY_Q5_0 = 8; - public static final int LLAMA_FTYPE_MOSTLY_Q5_1 = 9; - public static final int LLAMA_FTYPE_MOSTLY_Q2_K = 10; - public static final int LLAMA_FTYPE_MOSTLY_Q3_K_S = 11; - public static final int LLAMA_FTYPE_MOSTLY_Q3_K_M = 12; - public static final int LLAMA_FTYPE_MOSTLY_Q3_K_L = 13; - public static final int LLAMA_FTYPE_MOSTLY_Q4_K_S = 14; - public static final int LLAMA_FTYPE_MOSTLY_Q4_K_M = 15; - public static final int LLAMA_FTYPE_MOSTLY_Q5_K_S = 16; - public static final int LLAMA_FTYPE_MOSTLY_Q5_K_M = 17; - public static final int LLAMA_FTYPE_MOSTLY_Q6_K = 18; - public static final int LLAMA_FTYPE_GUESSED = 1024; - }; - public static interface llama_gretype { - public static final int LLAMA_GRETYPE_END = 0; - public static final int LLAMA_GRETYPE_ALT = 1; - public static final int LLAMA_GRETYPE_RULE_REF = 2; - public static final int LLAMA_GRETYPE_CHAR = 3; - public static final int LLAMA_GRETYPE_CHAR_NOT = 4; - public static final int LLAMA_GRETYPE_CHAR_RNG_UPPER = 5; - public static final int LLAMA_GRETYPE_CHAR_ALT = 6; - }; - public static final int LLAMA_MAX_DEVICES = (int)1; - public static final long LLAMA_DEFAULT_SEED = (long)0xFFFFFFFFL; - public static final int LLAMA_FILE_MAGIC_GGSN = (int)0x6767736e; - public static final int LLAMA_SESSION_MAGIC = (int)0x6767736e; - public static final int LLAMA_SESSION_VERSION = (int)1; - public interface llama_progress_callback extends Callback { - void apply(float progress, Pointer ctx); - }; - public interface llama_log_callback extends Callback { - void apply(int level, String text, Pointer user_data); - }; - public interface llama_beam_search_callback_fn_t extends Callback { - void apply(Pointer callback_data, de.kherud.llama.foreign.llama_beams_state.ByValue llama_beams_state1); - }; - public static native de.kherud.llama.foreign.llama_context_params.ByValue llama_context_default_params(); - public static native llama_model_quantize_params.ByValue llama_model_quantize_default_params(); - public static native void llama_backend_init(byte numa); - public static native void llama_backend_free(); - public static native LlamaLibrary.llama_model llama_load_model_from_file(String path_model, de.kherud.llama.foreign.llama_context_params.ByValue params); - public static native void llama_free_model(LlamaLibrary.llama_model model); - public static native LlamaLibrary.llama_context llama_new_context_with_model(LlamaLibrary.llama_model model, de.kherud.llama.foreign.llama_context_params.ByValue params); - public static native void llama_free(LlamaLibrary.llama_context ctx); - public static native long llama_time_us(); - public static native int llama_max_devices(); - public static native byte llama_mmap_supported(); - public static native byte llama_mlock_supported(); - public static native int llama_n_vocab(LlamaLibrary.llama_context ctx); - public static native int llama_n_ctx(LlamaLibrary.llama_context ctx); - public static native int llama_n_ctx_train(LlamaLibrary.llama_context ctx); - public static native int llama_n_embd(LlamaLibrary.llama_context ctx); - public static native int llama_vocab_type(LlamaLibrary.llama_context ctx); - public static native int llama_model_n_vocab(LlamaLibrary.llama_model model); - public static native int llama_model_n_ctx(LlamaLibrary.llama_model model); - public static native int llama_model_n_ctx_train(LlamaLibrary.llama_model model); - public static native int llama_model_n_embd(LlamaLibrary.llama_model model); - public static native int llama_model_desc(LlamaLibrary.llama_model model, byte[] buf, NativeSize buf_size); - public static native long llama_model_size(LlamaLibrary.llama_model model); - public static native long llama_model_n_params(LlamaLibrary.llama_model model); - public static native int llama_model_quantize(String fname_inp, String fname_out, llama_model_quantize_params params); - public static native int llama_apply_lora_from_file(LlamaLibrary.llama_context ctx, String path_lora, String path_base_model, int n_threads); - public static native int llama_model_apply_lora_from_file(LlamaLibrary.llama_model model, String path_lora, String path_base_model, int n_threads); - public static native int llama_get_kv_cache_token_count(LlamaLibrary.llama_context ctx); - public static native void llama_set_rng_seed(LlamaLibrary.llama_context ctx, int seed); - public static native NativeSize llama_get_state_size(LlamaLibrary.llama_context ctx); - public static native NativeSize llama_copy_state_data(LlamaLibrary.llama_context ctx, ByteBuffer dst); - public static native NativeSize llama_set_state_data(LlamaLibrary.llama_context ctx, ByteBuffer src); - public static native byte llama_load_session_file(LlamaLibrary.llama_context ctx, String path_session, IntBuffer tokens_out, NativeSize n_token_capacity, NativeSizeByReference n_token_count_out); - public static native byte llama_save_session_file(LlamaLibrary.llama_context ctx, String path_session, IntBuffer tokens, NativeSize n_token_count); - public static native int llama_eval(LlamaLibrary.llama_context ctx, IntBuffer tokens, int n_tokens, int n_past, int n_threads); - public static native int llama_eval_embd(LlamaLibrary.llama_context ctx, FloatBuffer embd, int n_tokens, int n_past, int n_threads); - public static native int llama_eval_export(LlamaLibrary.llama_context ctx, String fname); - public static native FloatByReference llama_get_logits(LlamaLibrary.llama_context ctx); - public static native FloatByReference llama_get_embeddings(LlamaLibrary.llama_context ctx); - public static native String llama_token_get_text(LlamaLibrary.llama_context ctx, int token); - public static native float llama_token_get_score(LlamaLibrary.llama_context ctx, int token); - public static native int llama_token_get_type(LlamaLibrary.llama_context ctx, int token); - public static native int llama_token_bos(LlamaLibrary.llama_context ctx); - public static native int llama_token_eos(LlamaLibrary.llama_context ctx); - public static native int llama_token_nl(LlamaLibrary.llama_context ctx); - public static native int llama_tokenize(LlamaLibrary.llama_context ctx, String text, IntBuffer tokens, int n_max_tokens, byte add_bos); - public static native int llama_tokenize_with_model(LlamaLibrary.llama_model model, String text, IntBuffer tokens, int n_max_tokens, byte add_bos); - public static native int llama_token_to_piece(LlamaLibrary.llama_context ctx, int token, byte[] buf, int length); - public static native int llama_token_to_piece_with_model(LlamaLibrary.llama_model model, int token, byte[] buf, int length); - public static native LlamaLibrary.llama_grammar llama_grammar_init(Pointer rules, NativeSize n_rules, NativeSize start_rule_index); - public static native void llama_grammar_free(LlamaLibrary.llama_grammar grammar); - public static native LlamaLibrary.llama_grammar llama_grammar_copy(LlamaLibrary.llama_grammar grammar); - public static native void llama_sample_repetition_penalty(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, IntBuffer last_tokens, NativeSize last_tokens_size, float penalty); - public static native void llama_sample_frequency_and_presence_penalties(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, IntBuffer last_tokens, NativeSize last_tokens_size, float alpha_frequency, float alpha_presence); - public static native void llama_sample_classifier_free_guidance(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, LlamaLibrary.llama_context guidance_ctx, float scale); - public static native void llama_sample_softmax(LlamaLibrary.llama_context ctx, llama_token_data_array candidates); - public static native void llama_sample_top_k(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, int k, NativeSize min_keep); - public static native void llama_sample_top_p(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, float p, NativeSize min_keep); - public static native void llama_sample_tail_free(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, float z, NativeSize min_keep); - public static native void llama_sample_typical(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, float p, NativeSize min_keep); - public static native void llama_sample_temperature(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, float temp); - public static native void llama_sample_grammar(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, LlamaLibrary.llama_grammar grammar); - public static native int llama_sample_token_mirostat(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, float tau, float eta, int m, FloatBuffer mu); - public static native int llama_sample_token_mirostat_v2(LlamaLibrary.llama_context ctx, llama_token_data_array candidates, float tau, float eta, FloatBuffer mu); - public static native int llama_sample_token_greedy(LlamaLibrary.llama_context ctx, llama_token_data_array candidates); - public static native int llama_sample_token(LlamaLibrary.llama_context ctx, llama_token_data_array candidates); - public static native void llama_grammar_accept_token(LlamaLibrary.llama_context ctx, LlamaLibrary.llama_grammar grammar, int token); - public static native void llama_beam_search(LlamaLibrary.llama_context ctx, LlamaLibrary.llama_beam_search_callback_fn_t callback, Pointer callback_data, NativeSize n_beams, int n_past, int n_predict, int n_threads); - public static native de.kherud.llama.foreign.llama_timings.ByValue llama_get_timings(LlamaLibrary.llama_context ctx); - public static native void llama_print_timings(LlamaLibrary.llama_context ctx); - public static native void llama_reset_timings(LlamaLibrary.llama_context ctx); - public static native String llama_print_system_info(); - public static native void llama_log_set(LlamaLibrary.llama_log_callback log_callback, Pointer user_data); - public static native void llama_dump_timing_info_yaml(PointerByReference stream, LlamaLibrary.llama_context ctx); - /** Pointer to unknown (opaque) type */ - public static class llama_grammar extends PointerType { - public llama_grammar(Pointer address) { - super(address); - } - public llama_grammar() { - super(); - } - }; - /** Pointer to unknown (opaque) type */ - public static class llama_model extends PointerType { - public llama_model(Pointer address) { - super(address); - } - public llama_model() { - super(); - } - }; - /** Pointer to unknown (opaque) type */ - public static class llama_context extends PointerType { - public llama_context(Pointer address) { - super(address); - } - public llama_context() { - super(); - } - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/NativeSize.java b/src/main/java/de/kherud/llama/foreign/NativeSize.java deleted file mode 100644 index a23fa30b2..000000000 --- a/src/main/java/de/kherud/llama/foreign/NativeSize.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2021 denkbares GmbH, Germany - * - * This is free software; you can redistribute it and/or modify it under the - * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) any - * later version. - * - * This software is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more - * details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this software; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF - * site: http://www.fsf.org. - */ - -package de.kherud.llama.foreign; - -import com.sun.jna.IntegerType; -import com.sun.jna.Native; - -public class NativeSize extends IntegerType { - private static final long serialVersionUID = 1L; - - public NativeSize() { - this(0); - } - - public NativeSize(long value) { - super(Native.SIZE_T_SIZE, value, true); - } - -} diff --git a/src/main/java/de/kherud/llama/foreign/NativeSizeByReference.java b/src/main/java/de/kherud/llama/foreign/NativeSizeByReference.java deleted file mode 100644 index 3d7876652..000000000 --- a/src/main/java/de/kherud/llama/foreign/NativeSizeByReference.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2021 denkbares GmbH, Germany - * - * This is free software; you can redistribute it and/or modify it under the - * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) any - * later version. - * - * This software is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more - * details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this software; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF - * site: http://www.fsf.org. - */ - -package de.kherud.llama.foreign; - -import com.sun.jna.Native; -import com.sun.jna.Pointer; -import com.sun.jna.ptr.ByReference; - -public class NativeSizeByReference extends ByReference { - public NativeSizeByReference() { - this(0); - } - - public NativeSizeByReference(long value) { - super(Native.SIZE_T_SIZE); - setValue(value); - } - - public void setValue(long value) { - Pointer p = getPointer(); - switch (Native.SIZE_T_SIZE) { - case 2: - p.setShort(0, (short)value); - break; - case 4: - p.setInt(0, (int)value); - break; - case 8: - p.setLong(0, value); - break; - default: - throw new RuntimeException("Unsupported size: " + Native.SIZE_T_SIZE); - } - } - - public long getValue() { - Pointer p = getPointer(); - switch (Native.SIZE_T_SIZE) { - case 2: - return p.getShort(0) & 0xFFFFL; - case 4: - return p.getInt(0) & 0xFFFFFFFFL; - case 8: - return p.getLong(0); - default: - throw new RuntimeException("Unsupported size: " + Native.SIZE_T_SIZE); - } - } -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_beam_view.java b/src/main/java/de/kherud/llama/foreign/llama_beam_view.java deleted file mode 100644 index 59c3462df..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_beam_view.java +++ /dev/null @@ -1,66 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import com.sun.jna.ptr.IntByReference; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_beam_view extends Structure { - public IntByReference tokens; - public IntByReference getTokens() { - return tokens; - } - public void setTokens(IntByReference tokens) { - this.tokens = tokens; - } - public NativeSize n_tokens; - public NativeSize getN_tokens() { - return n_tokens; - } - public void setN_tokens(NativeSize n_tokens) { - this.n_tokens = n_tokens; - } - /** Cumulative beam probability (renormalized relative to all beams) */ - public float p; - public float getP() { - return p; - } - public void setP(float p) { - this.p = p; - } - /** Callback should set this to true when a beam is at end-of-beam. */ - public byte eob; - public byte getEob() { - return eob; - } - public void setEob(byte eob) { - this.eob = eob; - } - public llama_beam_view() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("tokens", "n_tokens", "p", "eob"); - } - public llama_beam_view(IntByReference tokens, NativeSize n_tokens, float p, byte eob) { - super(); - this.tokens = tokens; - this.n_tokens = n_tokens; - this.p = p; - this.eob = eob; - } - public llama_beam_view(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_beam_view implements Structure.ByReference { - - }; - public static class ByValue extends llama_beam_view implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_beams_state.java b/src/main/java/de/kherud/llama/foreign/llama_beams_state.java deleted file mode 100644 index 2e9cd3ab2..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_beams_state.java +++ /dev/null @@ -1,66 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_beams_state extends Structure { - public de.kherud.llama.foreign.llama_beam_view.ByReference beam_views; - public de.kherud.llama.foreign.llama_beam_view.ByReference getBeam_views() { - return beam_views; - } - public void setBeam_views(de.kherud.llama.foreign.llama_beam_view.ByReference beam_views) { - this.beam_views = beam_views; - } - /** Number of elements in beam_views[]. */ - public NativeSize n_beams; - public NativeSize getN_beams() { - return n_beams; - } - public void setN_beams(NativeSize n_beams) { - this.n_beams = n_beams; - } - /** Current max length of prefix tokens shared by all beams. */ - public NativeSize common_prefix_length; - public NativeSize getCommon_prefix_length() { - return common_prefix_length; - } - public void setCommon_prefix_length(NativeSize common_prefix_length) { - this.common_prefix_length = common_prefix_length; - } - /** True iff this is the last callback invocation. */ - public byte last_call; - public byte getLast_call() { - return last_call; - } - public void setLast_call(byte last_call) { - this.last_call = last_call; - } - public llama_beams_state() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("beam_views", "n_beams", "common_prefix_length", "last_call"); - } - public llama_beams_state(de.kherud.llama.foreign.llama_beam_view.ByReference beam_views, NativeSize n_beams, NativeSize common_prefix_length, byte last_call) { - super(); - this.beam_views = beam_views; - this.n_beams = n_beams; - this.common_prefix_length = common_prefix_length; - this.last_call = last_call; - } - public llama_beams_state(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_beams_state implements Structure.ByReference { - - }; - public static class ByValue extends llama_beams_state implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_context_params.java b/src/main/java/de/kherud/llama/foreign/llama_context_params.java deleted file mode 100644 index b6a630bb3..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_context_params.java +++ /dev/null @@ -1,172 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import com.sun.jna.ptr.FloatByReference; -import de.kherud.llama.foreign.LlamaLibrary.llama_progress_callback; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_context_params extends Structure { - /** RNG seed, -1 for random */ - public int seed; - public int getSeed() { - return seed; - } - public void setSeed(int seed) { - this.seed = seed; - } - /** text context */ - public int n_ctx; - public int getN_ctx() { - return n_ctx; - } - public void setN_ctx(int n_ctx) { - this.n_ctx = n_ctx; - } - /** prompt processing batch size */ - public int n_batch; - public int getN_batch() { - return n_batch; - } - public void setN_batch(int n_batch) { - this.n_batch = n_batch; - } - /** number of layers to store in VRAM */ - public int n_gpu_layers; - public int getN_gpu_layers() { - return n_gpu_layers; - } - public void setN_gpu_layers(int n_gpu_layers) { - this.n_gpu_layers = n_gpu_layers; - } - /** the GPU that is used for scratch and small tensors */ - public int main_gpu; - public int getMain_gpu() { - return main_gpu; - } - public void setMain_gpu(int main_gpu) { - this.main_gpu = main_gpu; - } - /** how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES) */ - public FloatByReference tensor_split; - public FloatByReference getTensor_split() { - return tensor_split; - } - public void setTensor_split(FloatByReference tensor_split) { - this.tensor_split = tensor_split; - } - /** RoPE base frequency */ - public float rope_freq_base; - public float getRope_freq_base() { - return rope_freq_base; - } - public void setRope_freq_base(float rope_freq_base) { - this.rope_freq_base = rope_freq_base; - } - /** RoPE frequency scaling factor */ - public float rope_freq_scale; - public float getRope_freq_scale() { - return rope_freq_scale; - } - public void setRope_freq_scale(float rope_freq_scale) { - this.rope_freq_scale = rope_freq_scale; - } - public llama_progress_callback progress_callback; - public llama_progress_callback getProgress_callback() { - return progress_callback; - } - public void setProgress_callback(llama_progress_callback progress_callback) { - this.progress_callback = progress_callback; - } - public Pointer progress_callback_user_data; - public Pointer getProgress_callback_user_data() { - return progress_callback_user_data; - } - public void setProgress_callback_user_data(Pointer progress_callback_user_data) { - this.progress_callback_user_data = progress_callback_user_data; - } - /** if true, reduce VRAM usage at the cost of performance */ - public byte low_vram; - public byte getLow_vram() { - return low_vram; - } - public void setLow_vram(byte low_vram) { - this.low_vram = low_vram; - } - /** if true, use experimental mul_mat_q kernels */ - public byte mul_mat_q; - public byte getMul_mat_q() { - return mul_mat_q; - } - public void setMul_mat_q(byte mul_mat_q) { - this.mul_mat_q = mul_mat_q; - } - /** use fp16 for KV cache */ - public byte f16_kv; - public byte getF16_kv() { - return f16_kv; - } - public void setF16_kv(byte f16_kv) { - this.f16_kv = f16_kv; - } - /** the llama_eval() call computes all logits, not just the last one */ - public byte logits_all; - public byte getLogits_all() { - return logits_all; - } - public void setLogits_all(byte logits_all) { - this.logits_all = logits_all; - } - /** only load the vocabulary, no weights */ - public byte vocab_only; - public byte getVocab_only() { - return vocab_only; - } - public void setVocab_only(byte vocab_only) { - this.vocab_only = vocab_only; - } - /** use mmap if possible */ - public byte use_mmap; - public byte getUse_mmap() { - return use_mmap; - } - public void setUse_mmap(byte use_mmap) { - this.use_mmap = use_mmap; - } - /** force system to keep model in RAM */ - public byte use_mlock; - public byte getUse_mlock() { - return use_mlock; - } - public void setUse_mlock(byte use_mlock) { - this.use_mlock = use_mlock; - } - /** embedding mode only */ - public byte embedding; - public byte getEmbedding() { - return embedding; - } - public void setEmbedding(byte embedding) { - this.embedding = embedding; - } - public llama_context_params() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("seed", "n_ctx", "n_batch", "n_gpu_layers", "main_gpu", "tensor_split", "rope_freq_base", "rope_freq_scale", "progress_callback", "progress_callback_user_data", "low_vram", "mul_mat_q", "f16_kv", "logits_all", "vocab_only", "use_mmap", "use_mlock", "embedding"); - } - public llama_context_params(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_context_params implements Structure.ByReference { - - }; - public static class ByValue extends llama_context_params implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_grammar_element.java b/src/main/java/de/kherud/llama/foreign/llama_grammar_element.java deleted file mode 100644 index fcae95ace..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_grammar_element.java +++ /dev/null @@ -1,49 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_grammar_element extends Structure { - /** @see LlamaLibrary.llama_gretype */ - public int type; - public int getType() { - return type; - } - public void setType(int type) { - this.type = type; - } - /** Unicode code point or rule ID */ - public int value; - public int getValue() { - return value; - } - public void setValue(int value) { - this.value = value; - } - public llama_grammar_element() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("type", "value"); - } - public llama_grammar_element(int type, int value) { - super(); - this.type = type; - this.value = value; - } - public llama_grammar_element(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_grammar_element implements Structure.ByReference { - - }; - public static class ByValue extends llama_grammar_element implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_model_quantize_params.java b/src/main/java/de/kherud/llama/foreign/llama_model_quantize_params.java deleted file mode 100644 index fe72c3725..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_model_quantize_params.java +++ /dev/null @@ -1,79 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_model_quantize_params extends Structure { - /** number of threads to use for quantizing, if less than 0 will use std::thread::hardware_concurrency() */ - public int nthread; - public int getNthread() { - return nthread; - } - public void setNthread(int nthread) { - this.nthread = nthread; - } - /** - * @see LlamaLibrary.llama_ftype - * quantize to this llama_ftype - */ - public int ftype; - public int getFtype() { - return ftype; - } - public void setFtype(int ftype) { - this.ftype = ftype; - } - /** allow quantizing non-f32/f16 tensors */ - public byte allow_requantize; - public byte getAllow_requantize() { - return allow_requantize; - } - public void setAllow_requantize(byte allow_requantize) { - this.allow_requantize = allow_requantize; - } - /** quantize output.weight */ - public byte quantize_output_tensor; - public byte getQuantize_output_tensor() { - return quantize_output_tensor; - } - public void setQuantize_output_tensor(byte quantize_output_tensor) { - this.quantize_output_tensor = quantize_output_tensor; - } - /** only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored */ - public byte only_copy; - public byte getOnly_copy() { - return only_copy; - } - public void setOnly_copy(byte only_copy) { - this.only_copy = only_copy; - } - public llama_model_quantize_params() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("nthread", "ftype", "allow_requantize", "quantize_output_tensor", "only_copy"); - } - public llama_model_quantize_params(int nthread, int ftype, byte allow_requantize, byte quantize_output_tensor, byte only_copy) { - super(); - this.nthread = nthread; - this.ftype = ftype; - this.allow_requantize = allow_requantize; - this.quantize_output_tensor = quantize_output_tensor; - this.only_copy = only_copy; - } - public llama_model_quantize_params(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_model_quantize_params implements Structure.ByReference { - - }; - public static class ByValue extends llama_model_quantize_params implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_timings.java b/src/main/java/de/kherud/llama/foreign/llama_timings.java deleted file mode 100644 index ee4dcf54f..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_timings.java +++ /dev/null @@ -1,103 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_timings extends Structure { - public double t_start_ms; - public double getT_start_ms() { - return t_start_ms; - } - public void setT_start_ms(double t_start_ms) { - this.t_start_ms = t_start_ms; - } - public double t_end_ms; - public double getT_end_ms() { - return t_end_ms; - } - public void setT_end_ms(double t_end_ms) { - this.t_end_ms = t_end_ms; - } - public double t_load_ms; - public double getT_load_ms() { - return t_load_ms; - } - public void setT_load_ms(double t_load_ms) { - this.t_load_ms = t_load_ms; - } - public double t_sample_ms; - public double getT_sample_ms() { - return t_sample_ms; - } - public void setT_sample_ms(double t_sample_ms) { - this.t_sample_ms = t_sample_ms; - } - public double t_p_eval_ms; - public double getT_p_eval_ms() { - return t_p_eval_ms; - } - public void setT_p_eval_ms(double t_p_eval_ms) { - this.t_p_eval_ms = t_p_eval_ms; - } - public double t_eval_ms; - public double getT_eval_ms() { - return t_eval_ms; - } - public void setT_eval_ms(double t_eval_ms) { - this.t_eval_ms = t_eval_ms; - } - public int n_sample; - public int getN_sample() { - return n_sample; - } - public void setN_sample(int n_sample) { - this.n_sample = n_sample; - } - public int n_p_eval; - public int getN_p_eval() { - return n_p_eval; - } - public void setN_p_eval(int n_p_eval) { - this.n_p_eval = n_p_eval; - } - public int n_eval; - public int getN_eval() { - return n_eval; - } - public void setN_eval(int n_eval) { - this.n_eval = n_eval; - } - public llama_timings() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("t_start_ms", "t_end_ms", "t_load_ms", "t_sample_ms", "t_p_eval_ms", "t_eval_ms", "n_sample", "n_p_eval", "n_eval"); - } - public llama_timings(double t_start_ms, double t_end_ms, double t_load_ms, double t_sample_ms, double t_p_eval_ms, double t_eval_ms, int n_sample, int n_p_eval, int n_eval) { - super(); - this.t_start_ms = t_start_ms; - this.t_end_ms = t_end_ms; - this.t_load_ms = t_load_ms; - this.t_sample_ms = t_sample_ms; - this.t_p_eval_ms = t_p_eval_ms; - this.t_eval_ms = t_eval_ms; - this.n_sample = n_sample; - this.n_p_eval = n_p_eval; - this.n_eval = n_eval; - } - public llama_timings(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_timings implements Structure.ByReference { - - }; - public static class ByValue extends llama_timings implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_token_data.java b/src/main/java/de/kherud/llama/foreign/llama_token_data.java deleted file mode 100644 index 86a27c26b..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_token_data.java +++ /dev/null @@ -1,58 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_token_data extends Structure { - /** token id */ - public int id; - public int getId() { - return id; - } - public void setId(int id) { - this.id = id; - } - /** log-odds of the token */ - public float logit; - public float getLogit() { - return logit; - } - public void setLogit(float logit) { - this.logit = logit; - } - /** probability of the token */ - public float p; - public float getP() { - return p; - } - public void setP(float p) { - this.p = p; - } - public llama_token_data() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("id", "logit", "p"); - } - public llama_token_data(int id, float logit, float p) { - super(); - this.id = id; - this.logit = logit; - this.p = p; - } - public llama_token_data(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_token_data implements Structure.ByReference { - - }; - public static class ByValue extends llama_token_data implements Structure.ByValue { - - }; -} diff --git a/src/main/java/de/kherud/llama/foreign/llama_token_data_array.java b/src/main/java/de/kherud/llama/foreign/llama_token_data_array.java deleted file mode 100644 index a5320db5b..000000000 --- a/src/main/java/de/kherud/llama/foreign/llama_token_data_array.java +++ /dev/null @@ -1,55 +0,0 @@ -package de.kherud.llama.foreign; -import com.sun.jna.Pointer; -import com.sun.jna.Structure; -import java.util.Arrays; -import java.util.List; -/** - * This file was autogenerated by JNAerator,
- * a tool written by Olivier Chafik that uses a few opensource projects..
- * For help, please visit NativeLibs4Java , Rococoa, or JNA. - */ -public class llama_token_data_array extends Structure { - public de.kherud.llama.foreign.llama_token_data.ByReference data; - public de.kherud.llama.foreign.llama_token_data.ByReference getData() { - return data; - } - public void setData(de.kherud.llama.foreign.llama_token_data.ByReference data) { - this.data = data; - } - public NativeSize size; - public NativeSize getSize() { - return size; - } - public void setSize(NativeSize size) { - this.size = size; - } - public byte sorted; - public byte getSorted() { - return sorted; - } - public void setSorted(byte sorted) { - this.sorted = sorted; - } - public llama_token_data_array() { - super(); - } - @Override - protected List getFieldOrder() { - return Arrays.asList("data", "size", "sorted"); - } - public llama_token_data_array(de.kherud.llama.foreign.llama_token_data.ByReference data, NativeSize size, byte sorted) { - super(); - this.data = data; - this.size = size; - this.sorted = sorted; - } - public llama_token_data_array(Pointer peer) { - super(peer); - } - public static class ByReference extends llama_token_data_array implements Structure.ByReference { - - }; - public static class ByValue extends llama_token_data_array implements Structure.ByValue { - - }; -} diff --git a/src/test/java/de/kherud/llama/GrammarTest.java b/src/test/java/de/kherud/llama/GrammarTest.java deleted file mode 100644 index b70240207..000000000 --- a/src/test/java/de/kherud/llama/GrammarTest.java +++ /dev/null @@ -1,209 +0,0 @@ -package de.kherud.llama; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static de.kherud.llama.LlamaGrammar.Pair; -import static de.kherud.llama.foreign.LlamaLibrary.llama_gretype.*; - -public class GrammarTest { - - @Test - public void testParseGrammar1() { - String grammar = "root ::= (expr \"=\" term \"\\n\")+\n" + - "expr ::= term ([-+*/] term)*\n" + - "term ::= [0-9]+"; - - LlamaGrammar.ParseState state = new LlamaGrammar.ParseState(grammar); - - Map expectedIds = Map.of( - "expr", 2, - "expr_5", 5, - "expr_6", 6, - "root", 0, - "root_1", 1, - "root_4", 4, - "term", 3, - "term_7", 7 - ); - - Assert.assertEquals("unexpected amount of symbol ids", expectedIds.size(), state.symbol2Id.size()); - expectedIds.forEach((expectedKey, expectedValue) -> { - Integer actualValue = state.symbol2Id.get(expectedKey); - Assert.assertNotNull("expected key " + expectedKey + " missing ", actualValue); - Assert.assertEquals("values of rule " + expectedValue + " do not match", expectedValue, actualValue); - }); - - List> expectedRules = List.of( - new Pair<>(LLAMA_GRETYPE_RULE_REF, 4), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 2), - new Pair<>(LLAMA_GRETYPE_CHAR, 61), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_CHAR, 10), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 6), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 7), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 1), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 4), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 1), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 45), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 43), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 42), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 47), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 5), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 6), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 48), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 57), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 7), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 48), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 57), - new Pair<>(LLAMA_GRETYPE_END, 0) - ); - - - int index = 0; - for (List> rule : state.rules) { - for (Pair actualElement : rule) { - Pair expectedElement = expectedRules.get(index); - Assert.assertEquals("grammar element types do not match", expectedElement.a, actualElement.a); - Assert.assertEquals("grammar element values do not match", expectedElement.b, actualElement.b); - index++; - } - } - - Assert.assertFalse("parse state has no rules", state.rules.isEmpty()); - } - - @Test - public void testParseGrammar2() { - String grammar = "root ::= (expr \"=\" ws term \"\\n\")+\n" + - "expr ::= term ([-+*/] term)*\n" + - "term ::= ident | num | \"(\" ws expr \")\" ws\n" + - "ident ::= [a-z] [a-z0-9_]* ws\n" + - "num ::= [0-9]+ ws\n" + - "ws ::= [ \\t\\n]*"; - - LlamaGrammar.ParseState state = new LlamaGrammar.ParseState(grammar); - - Map expectedIds = new HashMap<>(); - expectedIds.put("expr", 2); - expectedIds.put("expr_6", 6); - expectedIds.put("expr_7", 7); - expectedIds.put("ident", 8); - expectedIds.put("ident_10", 10); - expectedIds.put("num", 9); - expectedIds.put("num_11", 11); - expectedIds.put("root", 0); - expectedIds.put("root_1", 1); - expectedIds.put("root_5", 5); - expectedIds.put("term", 4); - expectedIds.put("ws", 3); - expectedIds.put("ws_12", 12); - - Assert.assertEquals("unexpected amount of symbol ids", expectedIds.size(), state.symbol2Id.size()); - expectedIds.forEach((expectedKey, expectedValue) -> { - Integer actualValue = state.symbol2Id.get(expectedKey); - Assert.assertNotNull("expected key " + expectedKey + " missing ", actualValue); - Assert.assertEquals("values of rule " + expectedValue + " do not match", expectedValue, actualValue); - }); - - List> expectedRules = List.of( - new Pair<>(LLAMA_GRETYPE_RULE_REF, 5), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 2), - new Pair<>(LLAMA_GRETYPE_CHAR, 61), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 4), - new Pair<>(LLAMA_GRETYPE_CHAR, 10), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 4), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 7), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 12), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 8), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 9), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 40), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 2), - new Pair<>(LLAMA_GRETYPE_CHAR, 41), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 1), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 5), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 1), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 45), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 43), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 42), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 47), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 4), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 6), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 7), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 97), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 122), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 10), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 11), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 3), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 97), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 122), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 48), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 57), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 95), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 10), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 48), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 57), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 11), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 48), - new Pair<>(LLAMA_GRETYPE_CHAR_RNG_UPPER, 57), - new Pair<>(LLAMA_GRETYPE_END, 0), - new Pair<>(LLAMA_GRETYPE_CHAR, 32), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 9), - new Pair<>(LLAMA_GRETYPE_CHAR_ALT, 10), - new Pair<>(LLAMA_GRETYPE_RULE_REF, 12), - new Pair<>(LLAMA_GRETYPE_ALT, 0), - new Pair<>(LLAMA_GRETYPE_END, 0) - ); - - - int index = 0; - for (List> rule : state.rules) { - for (Pair actualElement : rule) { - Pair expectedElement = expectedRules.get(index); - Assert.assertEquals("grammar element types do not match", expectedElement.a, actualElement.a); - Assert.assertEquals("grammar element values do not match", expectedElement.b, actualElement.b); - index++; - } - } - - Assert.assertFalse("parse state has no rules", state.rules.isEmpty()); - } - -} diff --git a/src/test/java/de/kherud/llama/LlamaModelTest.java b/src/test/java/de/kherud/llama/LlamaModelTest.java new file mode 100644 index 000000000..e3e69d8c1 --- /dev/null +++ b/src/test/java/de/kherud/llama/LlamaModelTest.java @@ -0,0 +1,335 @@ +package de.kherud.llama; + +import java.io.*; +import java.util.*; +import java.util.regex.Pattern; + +import de.kherud.llama.args.LogFormat; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +public class LlamaModelTest { + + private static final String prefix = "def remove_non_ascii(s: str) -> str:\n \"\"\" "; + private static final String suffix = "\n return result\n"; + private static final int nPredict = 10; + + private static LlamaModel model; + + @BeforeClass + public static void setup() { +// LlamaModel.setLogger(LogFormat.TEXT, (level, msg) -> System.out.println(level + ": " + msg)); + model = new LlamaModel( + new ModelParameters() + .setCtxSize(128) + .setModel("models/codellama-7b.Q2_K.gguf") + //.setModelUrl("https://huggingface.co/TheBloke/CodeLlama-7B-GGUF/resolve/main/codellama-7b.Q2_K.gguf") + .setGpuLayers(43) + .enableEmbedding().enableLogTimestamps().enableLogPrefix() + ); + } + + @AfterClass + public static void tearDown() { + if (model != null) { + model.close(); + } + } + + @Test + public void testGenerateAnswer() { + Map logitBias = new HashMap<>(); + logitBias.put(2, 2.0f); + InferenceParameters params = new InferenceParameters(prefix) + .setTemperature(0.95f) + .setStopStrings("\"\"\"") + .setNPredict(nPredict) + .setTokenIdBias(logitBias); + + int generated = 0; + for (LlamaOutput ignored : model.generate(params)) { + generated++; + } + // todo: currently, after generating nPredict tokens, there is an additional empty output + Assert.assertTrue(generated > 0 && generated <= nPredict + 1); + } + + @Test + public void testGenerateInfill() { + Map logitBias = new HashMap<>(); + logitBias.put(2, 2.0f); + InferenceParameters params = new InferenceParameters("") + .setInputPrefix(prefix) + .setInputSuffix(suffix ) + .setTemperature(0.95f) + .setStopStrings("\"\"\"") + .setNPredict(nPredict) + .setTokenIdBias(logitBias) + .setSeed(42); + + int generated = 0; + for (LlamaOutput ignored : model.generate(params)) { + generated++; + } + Assert.assertTrue(generated > 0 && generated <= nPredict + 1); + } + + @Test + public void testGenerateGrammar() { + InferenceParameters params = new InferenceParameters("") + .setGrammar("root ::= (\"a\" | \"b\")+") + .setNPredict(nPredict); + StringBuilder sb = new StringBuilder(); + for (LlamaOutput output : model.generate(params)) { + sb.append(output); + } + String output = sb.toString(); + + Assert.assertTrue(output.matches("[ab]+")); + int generated = model.encode(output).length; + Assert.assertTrue(generated > 0 && generated <= nPredict + 1); + } + + @Test + public void testCompleteAnswer() { + Map logitBias = new HashMap<>(); + logitBias.put(2, 2.0f); + InferenceParameters params = new InferenceParameters(prefix) + .setTemperature(0.95f) + .setStopStrings("\"\"\"") + .setNPredict(nPredict) + .setTokenIdBias(logitBias) + .setSeed(42); + + String output = model.complete(params); + Assert.assertFalse(output.isEmpty()); + } + + @Test + public void testCompleteInfillCustom() { + Map logitBias = new HashMap<>(); + logitBias.put(2, 2.0f); + InferenceParameters params = new InferenceParameters("") + .setInputPrefix(prefix) + .setInputSuffix(suffix) + .setTemperature(0.95f) + .setStopStrings("\"\"\"") + .setNPredict(nPredict) + .setTokenIdBias(logitBias) + .setSeed(42); + + String output = model.complete(params); + Assert.assertFalse(output.isEmpty()); + } + + @Test + public void testCompleteGrammar() { + InferenceParameters params = new InferenceParameters("") + .setGrammar("root ::= (\"a\" | \"b\")+") + .setNPredict(nPredict); + String output = model.complete(params); + Assert.assertTrue(output + " doesn't match [ab]+", output.matches("[ab]+")); + int generated = model.encode(output).length; + Assert.assertTrue("generated count is: " + generated, generated > 0 && generated <= nPredict + 1); + + } + + @Test + public void testCancelGenerating() { + InferenceParameters params = new InferenceParameters(prefix).setNPredict(nPredict); + + int generated = 0; + LlamaIterator iterator = model.generate(params).iterator(); + while (iterator.hasNext()) { + iterator.next(); + generated++; + if (generated == 5) { + iterator.cancel(); + } + } + Assert.assertEquals(5, generated); + } + + @Test + public void testEmbedding() { + float[] embedding = model.embed(prefix); + Assert.assertEquals(4096, embedding.length); + } + + + @Ignore + /** + * To run this test download the model from here https://huggingface.co/mradermacher/jina-reranker-v1-tiny-en-GGUF/tree/main + * remove .enableEmbedding() from model setup and add .enableReRanking() and then enable the test. + */ + public void testReRanking() { + + String query = "Machine learning is"; + String [] TEST_DOCUMENTS = new String[] { + "A machine is a physical system that uses power to apply forces and control movement to perform an action. The term is commonly applied to artificial devices, such as those employing engines or motors, but also to natural biological macromolecules, such as molecular machines.", + "Learning is the process of acquiring new understanding, knowledge, behaviors, skills, values, attitudes, and preferences. The ability to learn is possessed by humans, non-human animals, and some machines; there is also evidence for some kind of learning in certain plants.", + "Machine learning is a field of study in artificial intelligence concerned with the development and study of statistical algorithms that can learn from data and generalize to unseen data, and thus perform tasks without explicit instructions.", + "Paris, capitale de la France, est une grande ville européenne et un centre mondial de l'art, de la mode, de la gastronomie et de la culture. Son paysage urbain du XIXe siècle est traversé par de larges boulevards et la Seine." + }; + LlamaOutput llamaOutput = model.rerank(query, TEST_DOCUMENTS[0], TEST_DOCUMENTS[1], TEST_DOCUMENTS[2], TEST_DOCUMENTS[3] ); + + System.out.println(llamaOutput); + } + + @Test + public void testTokenization() { + String prompt = "Hello, world!"; + int[] encoded = model.encode(prompt); + String decoded = model.decode(encoded); + // the llama tokenizer adds a space before the prompt + Assert.assertEquals(" " +prompt, decoded); + } + + @Ignore + public void testLogText() { + List messages = new ArrayList<>(); + LlamaModel.setLogger(LogFormat.TEXT, (level, msg) -> messages.add(new LogMessage(level, msg))); + + InferenceParameters params = new InferenceParameters(prefix) + .setNPredict(nPredict) + .setSeed(42); + model.complete(params); + + Assert.assertFalse(messages.isEmpty()); + + Pattern jsonPattern = Pattern.compile("^\\s*[\\[{].*[}\\]]\\s*$"); + for (LogMessage message : messages) { + Assert.assertNotNull(message.level); + Assert.assertFalse(jsonPattern.matcher(message.text).matches()); + } + } + + @Ignore + public void testLogJSON() { + List messages = new ArrayList<>(); + LlamaModel.setLogger(LogFormat.JSON, (level, msg) -> messages.add(new LogMessage(level, msg))); + + InferenceParameters params = new InferenceParameters(prefix) + .setNPredict(nPredict) + .setSeed(42); + model.complete(params); + + Assert.assertFalse(messages.isEmpty()); + + Pattern jsonPattern = Pattern.compile("^\\s*[\\[{].*[}\\]]\\s*$"); + for (LogMessage message : messages) { + Assert.assertNotNull(message.level); + Assert.assertTrue(jsonPattern.matcher(message.text).matches()); + } + } + + @Ignore + @Test + public void testLogStdout() { + // Unfortunately, `printf` can't be easily re-directed to Java. This test only works manually, thus. + InferenceParameters params = new InferenceParameters(prefix) + .setNPredict(nPredict) + .setSeed(42); + + System.out.println("########## Log Text ##########"); + LlamaModel.setLogger(LogFormat.TEXT, null); + model.complete(params); + + System.out.println("########## Log JSON ##########"); + LlamaModel.setLogger(LogFormat.JSON, null); + model.complete(params); + + System.out.println("########## Log None ##########"); + LlamaModel.setLogger(LogFormat.TEXT, (level, msg) -> {}); + model.complete(params); + + System.out.println("##############################"); + } + + private String completeAndReadStdOut() { + PrintStream stdOut = System.out; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + @SuppressWarnings("ImplicitDefaultCharsetUsage") PrintStream printStream = new PrintStream(outputStream); + System.setOut(printStream); + + try { + InferenceParameters params = new InferenceParameters(prefix) + .setNPredict(nPredict) + .setSeed(42); + model.complete(params); + } finally { + System.out.flush(); + System.setOut(stdOut); + printStream.close(); + } + + return outputStream.toString(); + } + + private List splitLines(String text) { + List lines = new ArrayList<>(); + + Scanner scanner = new Scanner(text); + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + lines.add(line); + } + scanner.close(); + + return lines; + } + + private static final class LogMessage { + private final LogLevel level; + private final String text; + + private LogMessage(LogLevel level, String text) { + this.level = level; + this.text = text; + } + } + + @Test + public void testJsonSchemaToGrammar() { + String schema = "{\n" + + " \"properties\": {\n" + + " \"a\": {\"type\": \"string\"},\n" + + " \"b\": {\"type\": \"string\"},\n" + + " \"c\": {\"type\": \"string\"}\n" + + " },\n" + + " \"additionalProperties\": false\n" + + "}"; + + String expectedGrammar = "a-kv ::= \"\\\"a\\\"\" space \":\" space string\n" + + "a-rest ::= ( \",\" space b-kv )? b-rest\n" + + "b-kv ::= \"\\\"b\\\"\" space \":\" space string\n" + + "b-rest ::= ( \",\" space c-kv )?\n" + + "c-kv ::= \"\\\"c\\\"\" space \":\" space string\n" + + "char ::= [^\"\\\\\\x7F\\x00-\\x1F] | [\\\\] ([\"\\\\bfnrt] | \"u\" [0-9a-fA-F]{4})\n" + + "root ::= \"{\" space (a-kv a-rest | b-kv b-rest | c-kv )? \"}\" space\n" + + "space ::= | \" \" | \"\\n\"{1,2} [ \\t]{0,20}\n" + + "string ::= \"\\\"\" char* \"\\\"\" space\n"; + + String actualGrammar = LlamaModel.jsonSchemaToGrammar(schema); + Assert.assertEquals(expectedGrammar, actualGrammar); + } + + @Test + public void testTemplate() { + + List> userMessages = new ArrayList<>(); + userMessages.add(new Pair<>("user", "What is the best book?")); + userMessages.add(new Pair<>("assistant", "It depends on your interests. Do you like fiction or non-fiction?")); + + InferenceParameters params = new InferenceParameters("A book recommendation system.") + .setMessages("Book", userMessages) + .setTemperature(0.95f) + .setStopStrings("\"\"\"") + .setNPredict(nPredict) + .setSeed(42); + Assert.assertEquals(model.applyTemplate(params), "<|im_start|>system\nBook<|im_end|>\n<|im_start|>user\nWhat is the best book?<|im_end|>\n<|im_start|>assistant\nIt depends on your interests. Do you like fiction or non-fiction?<|im_end|>\n<|im_start|>assistant\n"); + } +} diff --git a/src/test/java/de/kherud/llama/RerankingModelTest.java b/src/test/java/de/kherud/llama/RerankingModelTest.java new file mode 100644 index 000000000..60d32bde4 --- /dev/null +++ b/src/test/java/de/kherud/llama/RerankingModelTest.java @@ -0,0 +1,83 @@ +package de.kherud.llama; + +import java.util.List; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class RerankingModelTest { + + private static LlamaModel model; + + String query = "Machine learning is"; + String[] TEST_DOCUMENTS = new String[] { + "A machine is a physical system that uses power to apply forces and control movement to perform an action. The term is commonly applied to artificial devices, such as those employing engines or motors, but also to natural biological macromolecules, such as molecular machines.", + "Learning is the process of acquiring new understanding, knowledge, behaviors, skills, values, attitudes, and preferences. The ability to learn is possessed by humans, non-human animals, and some machines; there is also evidence for some kind of learning in certain plants.", + "Machine learning is a field of study in artificial intelligence concerned with the development and study of statistical algorithms that can learn from data and generalize to unseen data, and thus perform tasks without explicit instructions.", + "Paris, capitale de la France, est une grande ville européenne et un centre mondial de l'art, de la mode, de la gastronomie et de la culture. Son paysage urbain du XIXe siècle est traversé par de larges boulevards et la Seine." }; + + @BeforeClass + public static void setup() { + model = new LlamaModel( + new ModelParameters().setCtxSize(128).setModel("models/jina-reranker-v1-tiny-en-Q4_0.gguf") + .setGpuLayers(43).enableReranking().enableLogTimestamps().enableLogPrefix()); + } + + @AfterClass + public static void tearDown() { + if (model != null) { + model.close(); + } + } + + @Test + public void testReRanking() { + + + LlamaOutput llamaOutput = model.rerank(query, TEST_DOCUMENTS[0], TEST_DOCUMENTS[1], TEST_DOCUMENTS[2], + TEST_DOCUMENTS[3]); + + Map rankedDocumentsMap = llamaOutput.probabilities; + Assert.assertTrue(rankedDocumentsMap.size()==TEST_DOCUMENTS.length); + + // Finding the most and least relevant documents + String mostRelevantDoc = null; + String leastRelevantDoc = null; + float maxScore = Float.MIN_VALUE; + float minScore = Float.MAX_VALUE; + + for (Map.Entry entry : rankedDocumentsMap.entrySet()) { + if (entry.getValue() > maxScore) { + maxScore = entry.getValue(); + mostRelevantDoc = entry.getKey(); + } + if (entry.getValue() < minScore) { + minScore = entry.getValue(); + leastRelevantDoc = entry.getKey(); + } + } + + // Assertions + Assert.assertTrue(maxScore > minScore); + Assert.assertEquals("Machine learning is a field of study in artificial intelligence concerned with the development and study of statistical algorithms that can learn from data and generalize to unseen data, and thus perform tasks without explicit instructions.", mostRelevantDoc); + Assert.assertEquals("Paris, capitale de la France, est une grande ville européenne et un centre mondial de l'art, de la mode, de la gastronomie et de la culture. Son paysage urbain du XIXe siècle est traversé par de larges boulevards et la Seine.", leastRelevantDoc); + + + } + + @Test + public void testSortedReRanking() { + List> rankedDocuments = model.rerank(true, query, TEST_DOCUMENTS); + Assert.assertEquals(rankedDocuments.size(), TEST_DOCUMENTS.length); + + // Check the ranking order: each score should be >= the next one + for (int i = 0; i < rankedDocuments.size() - 1; i++) { + float currentScore = rankedDocuments.get(i).getValue(); + float nextScore = rankedDocuments.get(i + 1).getValue(); + Assert.assertTrue("Ranking order incorrect at index " + i, currentScore >= nextScore); + } + } +} diff --git a/src/test/java/examples/GrammarExample.java b/src/test/java/examples/GrammarExample.java index b45fd1397..d90de2066 100644 --- a/src/test/java/examples/GrammarExample.java +++ b/src/test/java/examples/GrammarExample.java @@ -1,27 +1,26 @@ package examples; +import de.kherud.llama.LlamaOutput; +import de.kherud.llama.ModelParameters; + +import de.kherud.llama.InferenceParameters; import de.kherud.llama.LlamaModel; -import de.kherud.llama.Parameters; public class GrammarExample { - public static void main(String... args) { - String grammar = "root ::= (expr \"=\" term \"\\n\")+\n" + - "expr ::= term ([-+*/] term)*\n" + - "term ::= [0-9]+"; - Parameters params = new Parameters.Builder() - .setNGpuLayers(43) - .setTemperature(0.7f) - .setMirostat(Parameters.MiroStat.V2) - .setGrammar(grammar) - .build(); - - String modelPath = "/Users/konstantin.herud/denkbares/projects/llama.cpp/models/13B/gguf-model-q4_0.bin"; - try (LlamaModel model = new LlamaModel(modelPath, params)) { - for (LlamaModel.Output output : model.generate("1+1=")) { - System.out.print(output); - } - } - } + public static void main(String... args) { + String grammar = "root ::= (expr \"=\" term \"\\n\")+\n" + + "expr ::= term ([-+*/] term)*\n" + + "term ::= [0-9]"; + ModelParameters modelParams = new ModelParameters() + .setModel("models/mistral-7b-instruct-v0.2.Q2_K.gguf"); + InferenceParameters inferParams = new InferenceParameters("") + .setGrammar(grammar); + try (LlamaModel model = new LlamaModel(modelParams)) { + for (LlamaOutput output : model.generate(inferParams)) { + System.out.print(output); + } + } + } } diff --git a/src/test/java/examples/InfillExample.java b/src/test/java/examples/InfillExample.java new file mode 100644 index 000000000..e13ecb7c0 --- /dev/null +++ b/src/test/java/examples/InfillExample.java @@ -0,0 +1,28 @@ +package examples; + +import de.kherud.llama.InferenceParameters; +import de.kherud.llama.LlamaModel; +import de.kherud.llama.LlamaOutput; +import de.kherud.llama.ModelParameters; + +public class InfillExample { + + public static void main(String... args) { + ModelParameters modelParams = new ModelParameters() + .setModel("models/codellama-7b.Q2_K.gguf") + .setGpuLayers(43); + + String prefix = "def remove_non_ascii(s: str) -> str:\n \"\"\" "; + String suffix = "\n return result\n"; + try (LlamaModel model = new LlamaModel(modelParams)) { + System.out.print(prefix); + InferenceParameters inferParams = new InferenceParameters("") + .setInputPrefix(prefix) + .setInputSuffix(suffix); + for (LlamaOutput output : model.generate(inferParams)) { + System.out.print(output); + } + System.out.print(suffix); + } + } +} diff --git a/src/test/java/examples/MainExample.java b/src/test/java/examples/MainExample.java index d550b790d..2b5150a52 100644 --- a/src/test/java/examples/MainExample.java +++ b/src/test/java/examples/MainExample.java @@ -1,42 +1,48 @@ package examples; -import de.kherud.llama.LlamaModel; -import de.kherud.llama.Parameters; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import de.kherud.llama.InferenceParameters; +import de.kherud.llama.LlamaModel; +import de.kherud.llama.LlamaOutput; +import de.kherud.llama.ModelParameters; +import de.kherud.llama.args.MiroStat; + +@SuppressWarnings("InfiniteLoopStatement") public class MainExample { public static void main(String... args) throws IOException { - Parameters params = new Parameters.Builder() - .setNGpuLayers(43) - .setTemperature(0.7f) - .setPenalizeNl(true) - .setMirostat(Parameters.MiroStat.V2) - .setAntiPrompt(new String[]{"\n"}) - .build(); - - String modelPath = "/run/media/konstantin/Seagate/models/llama2/llama-2-13b-chat/gguf-model-q4_0.bin"; + ModelParameters modelParams = new ModelParameters() + .setModel("models/mistral-7b-instruct-v0.2.Q2_K.gguf") + .setGpuLayers(43); String system = "This is a conversation between User and Llama, a friendly chatbot.\n" + "Llama is helpful, kind, honest, good at writing, and never fails to answer any " + - "requests immediately and with precision.\n"; + "requests immediately and with precision.\n\n" + + "User: Hello Llama\n" + + "Llama: Hello. How may I help you today?"; BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); - try (LlamaModel model = new LlamaModel(modelPath, params)) { + try (LlamaModel model = new LlamaModel(modelParams)) { + System.out.print(system); String prompt = system; while (true) { prompt += "\nUser: "; - System.out.print(prompt); + System.out.print("\nUser: "); String input = reader.readLine(); prompt += input; System.out.print("Llama: "); prompt += "\nLlama: "; - for (LlamaModel.Output output : model.generate(prompt)) { + InferenceParameters inferParams = new InferenceParameters(prompt) + .setTemperature(0.7f) + .setPenalizeNl(true) + .setMiroStat(MiroStat.V2) + .setStopStrings("User:"); + for (LlamaOutput output : model.generate(inferParams)) { System.out.print(output); + prompt += output; } - prompt = ""; } } }