diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..fcb76e26 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Scala Steward: Reformat with scalafmt 3.7.2 +2b37d1ce0bbed3d8d03a66292e7d90dcde805624 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..ac7ffb77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +sbt text eol=lf + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0f96f8de --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# Set update schedule for GitHub Actions + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..a145dbfe --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,55 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +categories: + - title: '🔥 Breaking Changes' + labels: + - 'breaking' + - title: '🚀 Features' + labels: + - 'feature' + - 'enhancement' + - title: '🐛 Bug Fixes' + labels: + - 'bug' + - title: '👋 Deprecated' + labels: + - 'deprecation' + - title: '🔗 Dependency Updates' + labels: + - 'library-update' + - 'dependencies' + - title: '🛠 Internal Updates' + labels: + - 'internal' + - 'kaizen' + - 'test-library-update' + - 'sbt-plugin-update' + - title: '📚 Docs' + labels: + - 'doc' +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' + +template: | + ## What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION + + +autolabeler: + - label: 'doc' + files: + - '*.md' + - label: 'feature' + title: + - '/(support|feature)/i' + - label: 'bug' + title: + - '/fix/i' + - label: 'internal' + title: + - '/internal/i' + - label: 'deprecation' + title: + - '/deprecate/i' diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..973c3db6 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,31 @@ +changelog: + categories: + - title: '🔥 Breaking Changes' + labels: + - 'breaking' + - title: '👋 Deprecated' + labels: + - 'deprecation' + - title: '🚀 Features' + labels: + - 'feature' + - 'enhancement' + - title: '🐛 Bug Fixes' + labels: + - 'bug' + - title: '🔗 Dependency Updates' + labels: + - 'library-update' + - 'dependencies' + - title: '🛠 Internal Updates' + labels: + - 'internal' + - 'kaizen' + - 'test-library-update' + - 'sbt-plugin-update' + - title: '📚 Docs' + labels: + - 'doc' + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml new file mode 100644 index 00000000..83e97ae9 --- /dev/null +++ b/.github/workflows/build-native.yml @@ -0,0 +1,47 @@ +name: Build Native + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'src/main/resources/org/xerial/snappy/VERSION' + - 'Makefile' + - 'Makefile.common' + - '**/*.h' + - '**/*.cpp' + - .github/workflows/build-native.yml + pull_request: + paths: + - 'src/main/resources/org/xerial/snappy/VERSION' + - 'Makefile' + - 'Makefile.common' + - '**/*.h' + - '**/*.cpp' + +jobs: + build: + permissions: + contents: write + pull-requests: write + name: Build native libraries + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v2.0 + with: + cmake-version: '3.16.x' + - name: Build native libraries + run: make clean-native native-all + env: + OCI_EXE: docker + - name: Create Pull Request + if: ${{ github.event_name != 'pull_request' }} + uses: peter-evans/create-pull-request@v8 + with: + title: Update native libraries + commit-message: Update native libraries for ${{ github.sha }} + branch: update-native-libs + labels: library-update diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 00000000..4023d3de --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,27 @@ +name: Release Drafter + +on: + # pull_request event is required only for autolabeler + pull_request: + # Only following types are handled by the action, but one can default to all as well + types: [opened, reopened, synchronize] + # pull_request_target event is required for autolabeler to support PRs from forks + pull_request_target: + types: [opened, reopened, synchronize] + +permissions: + contents: read + +jobs: + update_PR_labels: + permissions: + contents: read + # write permission is required for autolabeler + # otherwise, read permission is required at least + pull-requests: write + runs-on: ubuntu-latest + steps: + # Runs only pull-request labeler + - uses: release-drafter/release-drafter@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-note.yml b/.github/workflows/release-note.yml new file mode 100644 index 00000000..ac290e4c --- /dev/null +++ b/.github/workflows/release-note.yml @@ -0,0 +1,18 @@ +name: Release Note + +on: + push: + tags: + - v* + workflow_dispatch: + +jobs: + release: + name: Create a new release note + runs-on: ubuntu-latest + steps: + - name: Create a release note + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "$GITHUB_REF_NAME" --repo="$GITHUB_REPOSITORY" --generate-notes diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..6ac0cb70 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Release + +on: + push: + tags: + - v* + workflow_dispatch: + +jobs: + publish_jvm: + name: Publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 10000 + # Fetch all tags so that sbt-dynver can find the previous release version + - run: git fetch --tags -f + # Install OpenJDK 21 + - uses: actions/setup-java@v5 + with: + distribution: 'zulu' + java-version: '21' + - name: Setup GPG + env: + PGP_SECRET: ${{ secrets.PGP_SECRET }} + run: echo $PGP_SECRET | base64 --decode | gpg --import --batch --yes + - name: Build bundle + env: + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + run: | + ./sbt publishSigned + - name: Release to Sonatype + env: + SONATYPE_USERNAME: '${{ secrets.SONATYPE_USERNAME }}' + SONATYPE_PASSWORD: '${{ secrets.SONATYPE_PASSWORD }}' + run: ./sbt sonaRelease diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml new file mode 100644 index 00000000..1e851b7f --- /dev/null +++ b/.github/workflows/snapshot.yml @@ -0,0 +1,40 @@ +name: Snapshot Release + +on: + push: + branches: + - main + paths: + - '**.scala' + - '**.java' + - '**.sbt' + - 'src/main/resources/org/xerial/snappy/native/**' + - '.github/workflows/snapshot.yml' + tag: + - '!*' + workflow_dispatch: + +jobs: + publish_snapshots: + name: Publish snapshots + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 10000 + # Fetch all tags so that sbt-dynver can find the previous release version + - run: git fetch --tags + - uses: actions/setup-java@v5 + with: + distribution: 'zulu' + java-version: '21' + - uses: actions/cache@v5 + with: + path: ~/.cache + key: ${{ runner.os }}-snapshot-${{ hashFiles('**/*.sbt') }} + restore-keys: ${{ runner.os }}-snapshot- + - name: Publish snapshots + env: + SONATYPE_USERNAME: '${{ secrets.SONATYPE_USERNAME }}' + SONATYPE_PASSWORD: '${{ secrets.SONATYPE_PASSWORD }}' + run: ./sbt publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..0a9d7d39 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,67 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + code: ${{ steps.filter.outputs.code }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + code: + - '**.scala' + - '**.java' + - '**.sbt' + - '.github/workflows/*.yml' + - '**.so' + - '**.dll' + - 'src/main/resources/**' + - 'project/build.properties' + - '.scalafmt.conf' + - 'sbt' + - 'sbt.bat' + code_format: + name: code format + needs: changes + if: ${{ needs.changes.outputs.code == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: scalafmt test + run: ./sbt scalafmtCheckAll + test-jdk: + strategy: + matrix: + version: [ '8', '11', '17', '21', '25' ] + name: test jdk${{ matrix.version }} + needs: changes + if: ${{ needs.changes.outputs.code == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'zulu' + java-version: ${{ matrix.version }} + - uses: actions/cache@v5 + with: + path: ~/.cache + key: ${{ runner.os }}-jdk${{ matrix.version }}-${{ hashFiles('**/*.sbt') }} + restore-keys: ${{ runner.os }}-jdk${{ matrix.version }}- + - name: Test + run: | + if [ "${{ matrix.version }}" -ge "24" ]; then + export _JAVA_OPTIONS="--enable-native-access=ALL-UNNAMED" + fi + ./sbt test + - name: JAR Integration Test + run: ./script/test-jar-integration.sh diff --git a/.gitignore b/.gitignore index fc37fa82..f77b6bbf 100644 --- a/.gitignore +++ b/.gitignore @@ -17,9 +17,14 @@ lib_managed/ src_managed/ project/boot/ project/plugins/project/ +.bsp # Scala-IDE specific .scala_dependencies atlassian-ide-plugin.xml +src/main/java/**/*.h +.metals +.vscode +.bloop diff --git a/.scalafmt.conf b/.scalafmt.conf index bda502a5..39be2f71 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,3 +1,16 @@ -maxColumn = 180 +version = 3.9.4 +project.layout = StandardConvention +runner.dialect = scala3 +maxColumn = 100 style = defaultWithAlign -optIn.breaksInsideChains = true +docstrings.blankFirstLine = yes +rewrite.scala3.convertToNewSyntax = true +rewrite.scala3.removeOptionalBraces = yes +rewrite.scala3.insertEndMarkerMinLines = 30 +# Add a new line before case class +newlines.topLevelStatementBlankLines = [ + { + blanks { after = 1 } + } +] +newlines.source = unfold diff --git a/.travis.yml b/.travis.yml index 462474bb..326e2b2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,19 @@ -sudo: required +branches: + only: + - master -services: - - docker +language: java -language: scala +jdk: + - openjdk11 -scala: - - 2.12.4 +os: linux -jdk: - - oraclejdk8 +arch: + - s390x + - ppc64le script: - - ./script/travis-deploy.sh +# - sudo apt-get install -y openjdk-11-jdk; +# - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-s390x && export PATH=$JAVA_HOME/bin:$PATH; + - ./sbt test diff --git a/BUILD.md b/BUILD.md index 86eac4a9..f7bc226e 100755 --- a/BUILD.md +++ b/BUILD.md @@ -116,4 +116,9 @@ If you are using Mac and openjdk7 (or higher), use the following option: $ make native LIBNAME=libsnappyjava.dylib +## Building with Alpine +This project supports Alpine-based container images, which use the musl libc implementation. When building on Alpine, be aware that: +- All required dependencies are compatible with musl libc +- The build process has been tested and verified on Alpine environments +- No additional configuration is needed for Alpine compatibility diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..622b193c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,137 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +snappy-java is a Java port of Google's Snappy compression library, providing fast compression/decompression with JNI bindings for native performance across multiple platforms. + +## Build Commands + +### Using sbt (primary build tool) + +```bash +# Enter sbt console +./sbt + +# Run tests +./sbt test + +# Run tests matching a pattern +./sbt "testOnly *BitShuffleTest" + +# Create jar file +./sbt package + +# Publish to local Maven repository +./sbt publishM2 + +# Run tests continuously on code changes +./sbt "~test" +``` + +### Using make (for native library compilation) + +```bash +# Build native libraries +make + +# Clean build artifacts +make clean + +# Platform-specific builds (when cross-compiling) +make native-all +``` + +## Architecture + +### Core Components + +1. **Java API Layer** (`src/main/java/org/xerial/snappy/`) + - `Snappy.java`: Main API facade providing high-level compression/decompression methods + - `SnappyNative.java`: JNI interface to native Snappy library + - `SnappyLoader.java`: Handles platform-specific native library loading + +2. **Native Layer** (`src/main/java/org/xerial/snappy/`) + - `SnappyNative.cpp`: JNI implementation bridging Java and C++ Snappy + - `BitShuffleNative.cpp`: JNI implementation for BitShuffle algorithm + +3. **Stream API** (`src/main/java/org/xerial/snappy/`) + - `SnappyOutputStream`/`SnappyInputStream`: Block-based compression streams + - `SnappyFramedOutputStream`/`SnappyFramedInputStream`: Framing format implementation + - `SnappyHadoopCompatibleOutputStream`: Hadoop-compatible format + +4. **Memory Management** (`src/main/java/org/xerial/snappy/buffer/` and `/pool/`) + - Buffer allocation and pooling for efficient memory usage + - Direct ByteBuffer management for zero-copy operations + +### Platform Support + +The project includes pre-built native libraries for multiple platforms in `src/main/resources/org/xerial/snappy/native/`: +- Windows (x86, x86_64, aarch64) +- macOS (x86, x86_64, aarch64) +- Linux (x86, x86_64, aarch64, arm, armv6, armv7, ppc64, ppc64le, s390x, riscv64, loongarch64) +- AIX (ppc, ppc64) +- SunOS (x86, x86_64, sparc) +- Android (arm, aarch64) + +### Cross-compilation + +The project uses Docker-based cross-compilation toolchains (see `docker/` directory) for building native libraries across different architectures. + +## Testing + +```bash +# Run all tests with debug logging +./sbt "testOnly * -- -l debug" + +# Run specific test class +./sbt "testOnly org.xerial.snappy.SnappyTest" + +# The project uses AirSpec (for Scala tests) and JUnit (for Java tests) +# Tests are located in src/test/java/org/xerial/snappy/ +``` + +## Important Notes + +1. **Native Library Loading**: The project automatically detects the platform and loads the appropriate native library from resources +2. **Memory Safety**: Uses direct ByteBuffers for efficient memory operations - be aware of buffer boundaries +3. **Thread Safety**: Snappy compression/decompression methods are thread-safe as they don't maintain state +4. **OSGi Support**: The project includes OSGi bundle configuration in build.sbt +5. **Compatibility**: Multiple stream formats are supported - ensure you use matching read/write formats (see compatibility matrix in README.md) + +## Development Workflow + +### Pull Request Guidelines +- Use squashed commits when merging PRs to maintain clean commit history +- Run tests before creating PR: `./sbt test` +- Format code: `./sbt scalafmtAll` + +### Release Process +The project uses sbt's built-in Sonatype integration for publishing: + +```bash +# 1. Stage artifacts to Sonatype +./sbt publishSigned + +# 2. Release to Maven Central +./sbt sonaRelease +``` + +**Prerequisites:** +- Set up Sonatype credentials in `~/.sbt/1.0/sonatype.sbt`: + ```scala + credentials += Credentials("Sonatype Nexus Repository Manager", + "s01.oss.sonatype.org", + "", + "") + ``` +- Configure PGP signing for artifact signing + +### Common Development Tasks +- When making changes to native code, rebuild with `make clean-native native` +- For Java/Scala changes, use `./sbt ~test` for continuous testing +- Always test on multiple JDK versions if possible (8, 11, 17, 21) + +### Native Code Testing +- Use make clean-native native for testing native code diff --git a/Makefile b/Makefile index a05ddfb7..dc63ba92 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ include Makefile.common +$(info OS_NAME:$(OS_NAME), OS_ARCH:$(OS_ARCH)) + MVN:=mvn SBT:=./sbt @@ -8,11 +10,12 @@ all: snappy SNAPPY_OUT:=$(TARGET)/snappy-$(SNAPPY_VERSION)-$(os_arch) SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz -SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc +SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy-c.cc snappy.cc SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION) SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC)) SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy -SNAPPY_GIT_REV:=b02bfa754ebf27921d8da3bd2517eab445b84ff9 # 1.1.7 +# Change this variable to use a specific git commit +SNAPPY_GIT_REV:=$(SNAPPY_VERSION) SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log SNAPPY_CMAKE_CACHE=$(SNAPPY_OUT)/CMakeCache.txt @@ -77,6 +80,7 @@ $(SNAPPY_GIT_UNPACKED): @mkdir -p $(SNAPPY_SRC_DIR) git clone $(SNAPPY_GIT_REPO_URL) $(SNAPPY_SRC_DIR) git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(SNAPPY_VERSION) $(SNAPPY_GIT_REV) + cd $(SNAPPY_SRC_DIR) && git submodule update --init touch $@ $(SNAPPY_CMAKE_CACHE): $(SNAPPY_GIT_UNPACKED) @@ -90,24 +94,24 @@ snappy-header: $(SNAPPY_CMAKE_CACHE) $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java @mkdir -p $(TARGET)/jni-classes - $(JAVAC) -source 1.7 -target 1.7 -d $(TARGET)/jni-classes -sourcepath $(SRC) $< + $(JAVAC) -source 8 -target 8 -h $(SRC)/org/xerial/snappy/ -d $(TARGET)/jni-classes -sourcepath $(SRC) $< $(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class - $(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class: $(SRC)/org/xerial/snappy/BitShuffleNative.java @mkdir -p $(TARGET)/jni-classes - $(JAVAC) -source 1.7 -target 1.7 -d $(TARGET)/jni-classes -sourcepath $(SRC) $< + $(JAVAC) -source 8 -target 8 -h $(SRC)/org/xerial/snappy/ -d $(TARGET)/jni-classes -sourcepath $(SRC) $< $(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class - $(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.BitShuffleNative $(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED) # aarch64 can use big-endian optimzied code ifeq ($(OS_ARCH),aarch64) +ifeq ($(ENDIANESS),$(BIG_ENDIAN)) SNAPPY_CXX_OPTS:=-DSNAPPY_IS_BIG_ENDIAN endif +endif $(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc @mkdir -p $(@D) @@ -138,13 +142,26 @@ NATIVE_DIR:=src/main/resources/org/xerial/snappy/native/$(OS_NAME)/$(OS_ARCH) NATIVE_TARGET_DIR:=$(TARGET)/classes/org/xerial/snappy/native/$(OS_NAME)/$(OS_ARCH) NATIVE_DLL:=$(NATIVE_DIR)/$(LIBNAME) -snappy-jar-version:=snappy-java-$(shell perl -npe "s/version in ThisBuild\s+:=\s+\"(.*)\"/\1/" version.sbt | sed -e "/^$$/d") +snappy-jar-version:=snappy-java-$(shell ./script/dynver.sh | cut -d'=' -f2 | sed 's/[ \"]//g') + +jar-version: + echo $(snappy-jar-version) native: jni-header snappy-header $(NATIVE_DLL) native-nocmake: jni-header $(NATIVE_DLL) snappy: native $(TARGET)/$(snappy-jar-version).jar -native-all: win32 win64 mac64 native-arm linux32 linux64 linux-ppc64le linux-aarch64 +native-all: native native-arm clean-docker mac64 win32 win64 linux32 linux64 linux-ppc64le linux-riscv64 linux-s390x musl-image musl + +ifdef CI +# Clean docker images within CI to avoid no space left error +DOCKER_POST_PROCESS:=docker system prune --all --force --volumes +else +DOCKER_POST_PROCESS:= +endif + +clean-docker: + $(DOCKER_POST_PROCESS) $(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME) @mkdir -p $(@D) @@ -162,6 +179,7 @@ test: $(NATIVE_DLL) DOCKER_RUN_OPTS:=--rm + win32: jni-header ./docker/dockcross-windows-x86 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native snappy-header native CROSS_PREFIX=i686-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86 SNAPPY_CMAKE_OPTS="-DHAVE_SYS_UIO_H=0"' @@ -173,40 +191,60 @@ mac32: jni-header $(MAKE) native OS_NAME=Mac OS_ARCH=x86 mac64: jni-header - docker run -it $(DOCKER_RUN_OPTS) -v $$PWD:/workdir -e CROSS_TRIPLE=x86_64-apple-darwin multiarch/crossbuild make clean-native native OS_NAME=Mac OS_ARCH=x86_64 + docker run -i $(DOCKER_RUN_OPTS) -v $$PWD:/workdir -e CROSS_TRIPLE=x86_64-apple-darwin multiarch/crossbuild make clean-native native OS_NAME=Mac OS_ARCH=x86_64 linux32: jni-header - docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86' + docker run $(DOCKER_RUN_OPTS) -i -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86' linux64: jni-header - docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86_64' + docker run $(DOCKER_RUN_OPTS) -i -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86_64' + +musl-image: + docker build -f docker/Dockerfile.linux-x86_64-musl -t snappy-java-musl . + +musl: jni-header + docker run $(DOCKER_RUN_OPTS) -i -v $$PWD:/work snappy-java-musl bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86_64-musl' freebsd64: $(MAKE) native OS_NAME=FreeBSD OS_ARCH=x86_64 # For ARM -native-arm: linux-arm linux-armv6 linux-armv7 linux-android-arm +native-arm: linux-arm64 linux-android-arm linux-android-aarch64 linux-arm linux-armv6 linux-armv7 windows-aarch64 + +# TODO: CROSS_PREFIX can be replaced with ${CROSS_ROOT}/bin/${CROSS_TRIPLE}- in Makefile.common linux-arm: jni-header - ./docker/dockcross-armv5 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=arm-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm' + ./docker/dockcross-armv5 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/armv5-unknown-linux-gnueabi/bin//armv5-unknown-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm' linux-armv6: jni-header - ./docker/dockcross-armv6 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armv6' + ./docker/dockcross-armv6 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=armv6' linux-armv7: jni-header - ./docker/dockcross-armv7 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armv7' + ./docker/dockcross-armv7 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=armv7' linux-android-arm: jni-header ./docker/dockcross-android-arm -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/arm-linux-androideabi/bin/arm-linux-androideabi- OS_NAME=Linux OS_ARCH=android-arm' +linux-android-aarch64: jni-header + ./docker/dockcross-android-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=android-aarch64' + linux-ppc64le: jni-header - ./docker/dockcross-ppc64le -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64le-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64le' + ./docker/dockcross-ppc64le -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64le-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64le' linux-ppc64: jni-header - ./docker/dockcross-ppc64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64' + ./docker/dockcross-ppc64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64' + +linux-arm64: jni-header + ./docker/dockcross-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=aarch64-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=aarch64' + +linux-riscv64: jni-header + ./docker/dockcross-riscv64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/riscv64-unknown-linux-gnu/bin/riscv64-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=riscv64' + +linux-s390x: jni-header + ./docker/dockcross-s390x -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/s390x-ibm-linux-gnu/bin/s390x-ibm-linux-gnu- OS_NAME=Linux OS_ARCH=s390x' -linux-aarch64: jni-header - ./docker/dockcross-aarch64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=aarch64-linux-gnu- OS_NAME=Linux OS_ARCH=aarch64' +windows-aarch64: jni-header + ./docker/dockcross-windows-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/aarch64-w64-mingw32-cross/bin/aarch64-w64-mingw32- OS_NAME=Windows OS_ARCH=aarch64' javadoc: $(SBT) doc diff --git a/Makefile.common b/Makefile.common index 0e268111..6c2ac3c1 100755 --- a/Makefile.common +++ b/Makefile.common @@ -6,11 +6,12 @@ ifndef JAVA_HOME $(warning Set JAVA_HOME environment variable) endif - JAVA := "$$JAVA_HOME/bin/java" JAVAC := "$$JAVA_HOME/bin/javac" JAVAH := "$$JAVA_HOME/bin/javah" +LITTLE_ENDIAN := "Little Endian" +BIG_ENDIAN := "Big Endian" OSINFO_CLASS := org.xerial.snappy.OSInfo OSINFO_PROG := lib/org/xerial/snappy/OSInfo.class @@ -24,6 +25,12 @@ OS_ARCH := $(shell $(JAVA) -cp lib $(OSINFO_CLASS) --arch) LIB_FOLDER := $(shell $(JAVA) -cp lib $(OSINFO_CLASS)) IBM_JDK_LIB := lib/inc_ibm +ifeq ($(OS_NAME),Mac) +ENDIANESS:="Little Endian" +else +ENDIANESS:=$(shell lscpu | grep "Byte Order" |cut -d ":" -f2) +endif + # Windows uses different path separators ifeq ($(OS_NAME),Windows) sep := ; @@ -42,7 +49,8 @@ endif # os=Default is meant to be generic unix/linux -known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armv6 Linux-armv7 Linux-android-arm Linux-aarch64 Linux-ppc Linux-ppc64 Linux-ppc64le Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64 +# To support new CPU architecture, add a new target name here +known_os_archs := Linux-x86_64-musl Linux-x86 Linux-x86_64 Linux-arm Linux-armv6 Linux-armv7 Linux-android-arm Linux-android-aarch64 Linux-aarch64 Linux-ppc Linux-ppc64 Linux-ppc64le Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 Mac-aarch64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 Windows-aarch64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64 Linux-riscv Linux-riscv64 Linux-loongarch Linux-loongarch64 os_arch := $(OS_NAME)-$(OS_ARCH) IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7")) @@ -67,7 +75,7 @@ CROSS_PREFIX := Default_CXX := $(CROSS_PREFIX)g++ Default_STRIP := $(CROSS_PREFIX)strip -Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden +Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -std=c++11 Default_LINKFLAGS := -shared -static Default_LIBNAME := libsnappyjava.so Default_SNAPPY_FLAGS := @@ -75,9 +83,9 @@ Default_SNAPPY_FLAGS := Linux-x86_CXX := $(CROSS_PREFIX)g++ Linux-x86_STRIP := $(CROSS_PREFIX)strip ifeq ($(IBM_JDK_7),) - Linux-x86_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ + Linux-x86_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ -std=c++11 else - Linux-x86_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ + Linux-x86_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ -std=c++11 endif Linux-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-x86_LIBNAME := libsnappyjava.so @@ -86,20 +94,31 @@ Linux-x86_SNAPPY_FLAGS:= Linux-x86_64_CXX := $(CROSS_PREFIX)g++ Linux-x86_64_STRIP := $(CROSS_PREFIX)strip ifeq ($(IBM_JDK_7),) - Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 + Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 else - Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 + Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 endif Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-x86_64_LIBNAME := libsnappyjava.so Linux-x86_64_SNAPPY_FLAGS := +Linux-x86_64-musl_CXX := $(CROSS_PREFIX)g++ +Linux-x86_64-musl_STRIP := $(CROSS_PREFIX)strip +ifeq ($(IBM_JDK_7),) + Linux-x86_64-musl_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 +else + Linux-x86_64-musl_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 +endif +Linux-x86_64-musl_LINKFLAGS := -shared -static-libgcc -static-libstdc++ +Linux-x86_64-musl_LIBNAME := libsnappyjava.so +Linux-x86_64-musl_SNAPPY_FLAGS := + Linux-ppc_CXX := g++ Linux-ppc_STRIP := strip ifeq ($(IBM_JDK_7),) - Linux-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32 + Linux-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32 -std=c++11 else - Linux-ppc_CXXFLAGS := -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32 + Linux-ppc_CXXFLAGS := -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32 -std=c++11 endif Linux-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-ppc_LIBNAME := libsnappyjava.so @@ -108,20 +127,21 @@ Linux-ppc_SNAPPY_FLAGS := Linux-ppc64le_CXX := $(CROSS_PREFIX)g++ Linux-ppc64le_STRIP := $(CROSS_PREFIX)strip ifeq ($(IBM_JDK_7),) - Linux-ppc64le_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 + Linux-ppc64le_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 else - Linux-ppc64le_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 + Linux-ppc64le_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 -std=c++11 endif -Linux-ppc64le_LINKFLAGS := -shared -static-libgcc -static-libstdc++ +# ppcle64 GLIBC is at 2.17; so disable __tls_get_addr_opt which is dependent on 2.22; +Linux-ppc64le_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -Wl,--no-tls-optimize,--no-tls-get-addr-optimize Linux-ppc64le_LIBNAME := libsnappyjava.so Linux-ppc64le_SNAPPY_FLAGS := Linux-ppc64_CXX := $(CROSS_PREFIX)g++ Linux-ppc64_STRIP := $(CROSS_PREFIX)strip ifeq ($(IBM_JDK_7),) - Linux-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 + Linux-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 else - Linux-ppc64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 + Linux-ppc64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 -std=c++11 endif Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-ppc64_LIBNAME := libsnappyjava.so @@ -131,9 +151,9 @@ AIX-ppc_CXX := g++ AIX-ppc_STRIP := strip AIX-ppc_LIBNAME := libsnappyjava.a ifeq ($(IBM_JDK_7),) - AIX-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32 + AIX-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32 -std=c++11 else - AIX-ppc_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32 + AIX-ppc_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32 -std=c++11 endif AIX-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt AIX-ppc_SNAPPY_FLAGS := @@ -142,9 +162,9 @@ AIX-ppc64_CXX := g++ AIX-ppc64_STRIP := strip -X64 AIX-ppc64_LIBNAME := libsnappyjava.a ifeq ($(IBM_JDK_7),) - AIX-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64 + AIX-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64 -std=c++11 else - AIX-ppc64_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64 + AIX-ppc64_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64 -std=c++11 endif AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt AIX-ppc64_SNAPPY_FLAGS := @@ -152,20 +172,20 @@ AIX-ppc64_SNAPPY_FLAGS := Linux-s390_CXX := g++ Linux-s390_STRIP := strip ifeq ($(IBM_JDK_7),) - Linux-s390_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31 + Linux-s390_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31 -std=c++11 else - Linux-s390_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31 + Linux-s390_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31 -std=c++11 endif Linux-s390_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-s390_LIBNAME := libsnappyjava.so Linux-s390_SNAPPY_FLAGS := -Linux-s390x_CXX := g++ -Linux-s390x_STRIP := strip +Linux-s390x_CXX := $(CROSS_PREFIX)g++ +Linux-s390x_STRIP := $(CROSS_PREFIX)strip ifeq ($(IBM_JDK_7),) - Linux-s390x_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 + Linux-s390x_CXXFLAGS := -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 else - Linux-s390x_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64 + Linux-s390x_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64 -std=c++11 endif Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-s390x_LIBNAME := libsnappyjava.so @@ -199,74 +219,121 @@ SunOS-x86_64_SNAPPY_FLAGS := Linux-arm_CXX := $(CROSS_PREFIX)g++ Linux-arm_STRIP := $(CROSS_PREFIX)strip -Linux-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp +Linux-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp -std=c++11 Linux-arm_LINKFLAGS := -shared -static-libgcc Linux-arm_LIBNAME := libsnappyjava.so Linux-arm_SNAPPY_FLAGS:= -Linux-armv6_CXX := $(CROSS_PREFIX)g++ -Linux-armv6_STRIP := $(CROSS_PREFIX)strip -Linux-armv6_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard -Linux-armv6_LINKFLAGS := -shared -static-libgcc +Linux-armv6_CXX := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ +Linux-armv6_STRIP := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip +Linux-armv6_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard -std=c++11 +Linux-armv6_LINKFLAGS := -shared -static-libgcc -std=c++11 Linux-armv6_LIBNAME := libsnappyjava.so Linux-armv6_SNAPPY_FLAGS:= -Linux-armv7_CXX := $(CROSS_PREFIX)g++ -Linux-armv7_STRIP := $(CROSS_PREFIX)strip -Linux-armv7_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard +Linux-armv7_CXX := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ +Linux-armv7_STRIP := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip +Linux-armv7_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp -std=c++11 Linux-armv7_LINKFLAGS := -shared -static-libgcc Linux-armv7_LIBNAME := libsnappyjava.so Linux-armv7_SNAPPY_FLAGS:= Linux-android-arm_CXX := $(CROSS_PREFIX)g++ -Linux-android-arm_STRIP := $(CROSS_PREFIX)strip -Linux-android-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden +Linux-android-arm_STRIP := $(CROSS_ROOT)/bin/llvm-strip +Linux-android-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 Linux-android-arm_LINKFLAGS := -shared -static-libgcc Linux-android-arm_LIBNAME := libsnappyjava.so Linux-android-arm_SNAPPY_FLAGS:= +Linux-android-aarch64_CXX := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ +Linux-android-aarch64_STRIP := ${CROSS_ROOT}/bin/llvm-strip +Linux-android-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 +Linux-android-aarch64_LINKFLAGS := -shared -static-libgcc +Linux-android-aarch64_LIBNAME := libsnappyjava.so +Linux-android-aarch64_SNAPPY_FLAGS:= + Linux-aarch64_CXX := $(CROSS_PREFIX)g++ Linux-aarch64_STRIP := $(CROSS_PREFIX)strip -Linux-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -Linux-aarch64_LINKFLAGS := -shared -static-libgcc +Linux-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 +Linux-aarch64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-aarch64_LIBNAME := libsnappyjava.so Linux-aarch64_SNAPPY_FLAGS:= Mac-x86_CXX := g++ -arch i386 Mac-x86_STRIP := strip -x -Mac-x86_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.4 -fvisibility=hidden +Mac-x86_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.4 -fvisibility=hidden -std=c++11 Mac-x86_LINKFLAGS := -dynamiclib -static-libgcc -Mac-x86_LIBNAME := libsnappyjava.jnilib +Mac-x86_LIBNAME := libsnappyjava.dylib Mac-x86_SNAPPY_FLAGS := Mac-x86_64_CXX := c++ -arch $(OS_ARCH) Mac-x86_64_STRIP := strip -x -Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.5 -fvisibility=hidden +Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.7 -fvisibility=hidden -stdlib=libc++ -std=c++11 -ftemplate-depth=1024 Mac-x86_64_LINKFLAGS := -dynamiclib -Mac-x86_64_LIBNAME := libsnappyjava.jnilib +Mac-x86_64_LIBNAME := libsnappyjava.dylib Mac-x86_64_SNAPPY_FLAGS := +Mac-aarch64_CXX := c++ -arch arm64 +Mac-aarch64_STRIP := strip -x +Mac-aarch64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.7 -fvisibility=hidden -stdlib=libc++ -std=c++11 +Mac-aarch64_LINKFLAGS := -dynamiclib +Mac-aarch64_LIBNAME := libsnappyjava.dylib + FreeBSD-x86_64_CXX := $(CROSS_PREFIX)g++ FreeBSD-x86_64_STRIP := $(CROSS_PREFIX)strip -FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden +FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -std=c++11 FreeBSD-x86_64_LINKFLAGS := -shared -static-libgcc FreeBSD-x86_64_LIBNAME := libsnappyjava.so FreeBSD-x86_64_SNAPPY_FLAGS := Windows-x86_CXX := $(CROSS_PREFIX)g++ Windows-x86_STRIP := $(CROSS_PREFIX)strip -Windows-x86_CXXFLAGS := -Ilib/inc_win -O2 +Windows-x86_CXXFLAGS := -Ilib/inc_win -O2 -std=c++11 Windows-x86_LINKFLAGS := -Wl,--kill-at -shared -static Windows-x86_LIBNAME := snappyjava.dll Windows-x86_SNAPPY_FLAGS := Windows-x86_64_CXX := $(CROSS_PREFIX)g++ Windows-x86_64_STRIP := $(CROSS_PREFIX)strip -Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2 +Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2 -std=c++11 Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static Windows-x86_64_LIBNAME := snappyjava.dll Windows-x86_64_SNAPPY_FLAGS := +Linux-riscv_CXX := $(CROSS_PREFIX)g++ +Linux-riscv_STRIP := $(CROSS_PREFIX)strip +Linux-riscv_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 +Linux-riscv_LINKFLAGS := -shared -static-libgcc -static-libstdc++ +Linux-riscv_LIBNAME := libsnappyjava.so +Linux-riscv_SNAPPY_FLAGS:= + +Linux-riscv64_CXX := $(CROSS_PREFIX)g++ +Linux-riscv64_STRIP := $(CROSS_PREFIX)strip +Linux-riscv64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 +Linux-riscv64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ +Linux-riscv64_LIBNAME := libsnappyjava.so +Linux-riscv64_SNAPPY_FLAGS:= + +Windows-aarch64_CXX := $(CROSS_PREFIX)g++ +Windows-aarch64_STRIP := $(CROSS_PREFIX)strip +Windows-aarch64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -march=armv8-a -fvisibility=hidden -D_WIN32 -DNOMINMAX -std=c++11 +Windows-aarch64_LINKFLAGS := -Wl,--kill-at -shared -static +Windows-aarch64_LIBNAME := snappyjava.dll + +Linux-loongarch_CXX := $(CROSS_PREFIX)g++ +Linux-loongarch_STRIP := $(CROSS_PREFIX)strip +Linux-loongarch_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 +Linux-loongarch_LINKFLAGS := -shared +Linux-loongarch_LIBNAME := libsnappyjava.so +Linux-loongarch_SNAPPY_FLAGS:= + +Linux-loongarch64_CXX := $(CROSS_PREFIX)g++ +Linux-loongarch64_STRIP := $(CROSS_PREFIX)strip +Linux-loongarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11 +Linux-loongarch64_LINKFLAGS := -shared +Linux-loongarch64_LIBNAME := libsnappyjava.so +Linux-loongarch64_SNAPPY_FLAGS:= + CXX := $($(os_arch)_CXX) STRIP := $($(os_arch)_STRIP) @@ -277,7 +344,7 @@ SNAPPY_FLAGS := $($(os_arch)_SNAPPY_FLAGS) -CXXFLAGS := $(CXXFLAGS) -DHAVE_CONFIG_H +CXXFLAGS := $(CXXFLAGS) -DHAVE_CONFIG_H -DNDEBUG ifneq ($(jni_include),) CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)" endif diff --git a/Makefile.package b/Makefile.package index 3382c82a..6617e0ef 100755 --- a/Makefile.package +++ b/Makefile.package @@ -28,8 +28,8 @@ DLL_DIR=src/main/resources/org/xerial/snappy/native DLL_WIN=$(DLL_DIR)/Windows/x86/snappyjava.dll DLL_WIN64=$(DLL_DIR)/Windows/amd64/snappyjava.dll -DLL_MAC=$(DLL_DIR)/Mac/x86_64/libsnappyjava.jnilib -DLL_TIGER_MAC=$(DLL_DIR)/Mac/i386/libsnappyjava.jnilib +DLL_MAC=$(DLL_DIR)/Mac/x86_64/libsnappyjava.dylib +DLL_TIGER_MAC=$(DLL_DIR)/Mac/i386/libsnappyjava.dylib DLL_LINUX=$(DLL_DIR)/Linux/i386/libsnappyjava.so DLL_AMD64=$(DLL_DIR)/Linux/amd64/libsnappyjava.so diff --git a/Milestone.md b/Milestone.md index 782684ab..8344f8e7 100644 --- a/Milestone.md +++ b/Milestone.md @@ -1,5 +1,51 @@ + +Since snappy-java 1.1.9.0, release notes will be available in the GitHub releases page https://github.com/xerial/snappy-java/releases + Since version 1.1.0.x, Java 6 (1.6) or higher is required. +## snappy-java-1.1.8.3 (2021-01-20) + * Make pure-java Snappy thread-safe [#271](https://github.com/xerial/snappy-java/pull/271) + * Improved SnappyFramedInput/OutputStream performance by using java.util.zip.CRC32C [#269](https://github.com/xerial/snappy-java/pull/269) + +## snappy-java-1.1.8.2 (2020-11-28) + * Support Apple Silicon (M1, Mac-aarch64) + * Fixed the pure-java Snappy fallback logic when no native library for your platform is found. + +## snappy-java-1.1.8.1 (2020-11-09) + * Fixed an initialization issue when using a recent Mac OS X version [#265](https://github.com/xerial/snappy-java/pull/265) + +## snappy-java-1.1.8 (2020-10-20) + * Upgrade to [Snappy 1.1.8](https://github.com/google/snappy/releases/tag/1.1.8) with small performance improvements. + +## snappy-java-1.1.7.8 (2020-10-20) + * Big-endian support for pure-java Snappy implementation + * linux-aarch64 (arm64) binary embeds libstdc++ for portability + * internal: Fix make native-all target to support the latest version of dockcross + +## snappy-java-1.1.7.7 (2020-08-25) + * Built snappy-java with jdk8 to resolve #251 (java.lang.NoSuchMethodError) + +## snappy-java-1.1.7.6 (2020-06-26) + * Added an experimental support of pure-java Snappy https://github.com/xerial/snappy-java#using-pure-java-snappy-implementation + * Pure-java snappy doesn't support Snappy.isValidCompressedBuffer methods, but the other methods, Snappy.compress, uncompress, SnappyInput/OutputStream, SnappyFramedInput/OutputStream, etc., should work as expected. + * Changed the minimum JVM requirement to JDK 1.8 + +## snappy-java-1.1.7.5 (2020-05-06) + * Fixes java.lang.NoClassDefFoundError: org/xerial/snappy/pool/DefaultPoolFactory in 1.1.7.4 + +## snapy-java-1.1.7.4 (2020-05-05) + * __DO NOT USE THIS VERSION__ 1.1.7.4 misses a package for using SnappyFramed streams. + * Caching internal buffers for SnappyFramed streams [#234](https://github.com/xerial/snappy-java/pull/234) + * Fixed the native lib for ppc64le to work with glibc 2.17 (Previously it depended on 2.22) + +## snappy-java-1.1.7.3 (2019-03-25) + * Minor release + * Output the snappy header even for the empty input to address Spark's [issue](https://issues.apache.org/jira/browse/SPARK-27267) + * Fixed SnappyFramed stream to support Java 9 + +## snappy-java-1.1.7.2 (2018-05-21) + * Fix for aarch64 endian issue + ## snappy-java-1.1.7.1 (2017-12-07) * Fix for Android. No need to upgrade to this version if you are not using Android diff --git a/NOTICE b/NOTICE index 7f27722e..4e457052 100755 --- a/NOTICE +++ b/NOTICE @@ -5,8 +5,8 @@ This product includes software developed by Apache PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ (Apache 2.0 license) -This library containd statically linked libstdc++. This inclusion is allowed by -"GCC RUntime Library Exception" +This library contains statically linked libstdc++. This inclusion is allowed by +"GCC Runtime Library Exception" http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html == Contributors == @@ -15,3 +15,8 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html * Alec Wysoker * Performance and memory usage improvement +Third-Party Notices and Licenses: + +- Hadoop: Apache Hadoop is used as a dependency + License: Apache License 2.0 + Source/Reference: https://github.com/apache/hadoop/blob/trunk/NOTICE.txt diff --git a/README.md b/README.md index 8f0b24f1..bfc479df 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ -snappy-java [![Build Status](https://travis-ci.org/xerial/snappy-java.svg?branch=master)](https://travis-ci.org/xerial/snappy-java) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [![Javadoc](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java.svg?label=javadoc)](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java) [![Reference Status](https://www.versioneye.com/java/org.xerial.snappy:snappy-java/reference_badge.svg?style=flat-square)](https://www.versioneye.com/java/org.xerial.snappy:snappy-java/references) +snappy-java +[![Build Status](https://travis-ci.org/xerial/snappy-java.svg?branch=master)](https://travis-ci.org/xerial/snappy-java) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) +[![Javadoc](https://javadoc.io/badge2/org.xerial.snappy/snappy-java/javadoc.svg)](https://javadoc.io/doc/org.xerial.snappy/snappy-java) === -snappy-java is a Java port of the snappy -, a fast C++ compresser/decompresser developed by Google. +snappy-java is a Java port of the [snappy](https://github.com/google/snappy), a fast C++ compresser/decompresser developed by Google. ## Features * Fast compression/decompression around 200~400MB/sec. @@ -9,8 +11,8 @@ snappy-java is a Java port of the snappy * JNI-based implementation to achieve comparable performance to the native C++ version. * Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.). * Compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.) - * To improve the compression ratios of these arrays, you can use a fast data-rearrangement implementation ([`BitShuffle`](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.3-M1/snappy-java-1.1.3-M1-javadoc.jar/!/org/xerial/snappy/BitShuffle.html)) before compression - * Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`). + * To improve the compression ratios of these arrays, you can use a fast data-rearrangement implementation ([`BitShuffle`](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.8/snappy-java-1.1.8-javadoc.jar/!/org/xerial/snappy/BitShuffle.html)) before compression + * Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux, etc. snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`). * Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`. * [Framing-format support](https://github.com/google/snappy/blob/master/framing_format.txt) (Since 1.1.0 version) * OSGi support @@ -22,24 +24,24 @@ snappy-java is a Java port of the snappy * Here are some [benchmark results](https://github.com/ning/jvm-compressor-benchmark/wiki), comparing snappy-java and the other compressors `LZO-java`/`LZF`/`QuickLZ`/`Gzip`/`Bzip2`. Thanks [Tatu Saloranta @cotowncoder](http://twitter.com/#!/cowtowncoder) for providing the benchmark suite. - * The benchmark result indicates snappy-java is the fastest compreesor/decompressor in Java: http://ning.github.com/jvm-compressor-benchmark/results/canterbury-roundtrip-2011-07-28/index.html - * The decompression speed is twice as fast as the others: http://ning.github.com/jvm-compressor-benchmark/results/canterbury-uncompress-2011-07-28/index.html + * The benchmark result indicates snappy-java is the fastest compressor/decompressor in Java: https://ning.github.io/jvm-compressor-benchmark/results/canterbury-roundtrip-2011-07-28/index.html + * The decompression speed is twice as fast as the others: https://ning.github.io/jvm-compressor-benchmark/results/canterbury-uncompress-2011-07-28/index.html ## Download - [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [![Javadoc](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java.svg?label=javadoc)](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) +[![Javadoc](https://javadoc.io/badge2/org.xerial.snappy/snappy-java/javadoc.svg)](https://javadoc.io/doc/org.xerial.snappy/snappy-java) * [Release Notes](Milestone.md) The current stable version is available from here: - * Release version: http://central.maven.org/maven2/org/xerial/snappy/snappy-java/ + * Release version: https://repo1.maven.org/maven2/org/xerial/snappy/snappy-java/ * Snapshot version (the latest beta version): https://oss.sonatype.org/content/repositories/snapshots/org/xerial/snappy/snappy-java/ ### Using with Maven - * Snappy-java is available from Maven's central repository: - -Add the following dependency to your pom.xml: + +Snappy-java is available from Maven's central repository. Add the following dependency to your pom.xml: org.xerial.snappy @@ -49,12 +51,75 @@ Add the following dependency to your pom.xml: compile +### Using with Gradle + +``` +implementation("org.xerial.snappy:snappy-java:(version)") +``` + ### Using with sbt ``` libraryDependencies += "org.xerial.snappy" % "snappy-java" % "(version)" ``` +## JDK 24+ Native Access Requirements + +Starting with JDK 24, Java has introduced restrictions on native method access through [JEP 472: Prepare to Restrict the Use of JNI](https://openjdk.org/jeps/472). Since snappy-java uses JNI to load native libraries for high-performance compression, applications running on **JDK 24 or later** must enable native access. + +### Required JVM Flag + +When running on JDK 24+, add the following JVM flag to your application: + +```bash +--enable-native-access=ALL-UNNAMED +``` + +### Examples + +**Running a JAR:** +```bash +java --enable-native-access=ALL-UNNAMED -jar your-application.jar +``` + +**Maven:** +```xml + + org.apache.maven.plugins + maven-surefire-plugin + + --enable-native-access=ALL-UNNAMED + + +``` + +**Gradle:** +```gradle +tasks.withType(Test) { + jvmArgs '--enable-native-access=ALL-UNNAMED' +} +``` + +**sbt:** +```scala +javaOptions += "--enable-native-access=ALL-UNNAMED" +``` + +### Why is this needed? + +Per JEP 472's policy of "integrity by default," it is the application developer's responsibility (not the library's) to explicitly enable native access. This change improves security by making native operations visible and controlled at the application level. + +Without this flag on JDK 24+, you will see warnings like: +``` +WARNING: A restricted method in java.lang.System has been called +WARNING: java.lang.System::load has been called by org.xerial.snappy.SnappyLoader +WARNING: Use --enable-native-access=ALL-UNNAMED to avoid a warning for callers in this module +``` + +These warnings will become errors in future JDK releases. + +**Note:** This requirement only applies to JDK 24 and later. Earlier JDK versions (8-23) do not require this flag. + ## Usage First, import `org.xerial.snapy.Snappy` in your Java code: @@ -83,9 +148,20 @@ Stream-based compressor/decompressor `SnappyOutputStream`/`SnappyInputStream` ar * See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.3-M1/snappy-java-1.1.3-M1-javadoc.jar/!/index.html) #### Compatibility Notes - * `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`. Here is the data format compatibility matrix: + +The original Snappy format definition did not define a file format. It later added +a "framing" format to define a file format, but by this point major software was +already using an industry standard instead -- represented in this library by the +`SnappyOutputStream` and `SnappyInputStream` methods. + +For interoperability with other libraries, check that compatible formats are used. +Note that not all libraries support all variants. + + * `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`. * `SnappyHadoopCompatibleOutputStream` does not emit a file header but write out the current block size as a preemble to each block +#### Data format compatibility matrix: + | Write\Read | `Snappy.uncompress` | `SnappyInputStream` | `SnappyFramedInputStream` | `org.apache.hadoop.io.compress.SnappyCodec` | | --------------- |:-------------------:|:------------------:|:-----------------------:|:-------------------------------------------:| | `Snappy.compress` | ok | ok | x | x | @@ -130,7 +206,7 @@ snappy-java uses sbt (simple build tool for Scala) as a build tool. Here is a si $ ./sbt # enter sbt console > ~test # run tests upon source code change - > ~test-only * # run tests that matches a given name pattern + > ~testOnly # run tests that matches a given name pattern > publishM2 # publish jar to $HOME/.m2/repository > package # create jar file > findbugs # Produce findbugs report in target/findbugs @@ -145,6 +221,7 @@ $ ./sbt -Dloglevel=debug For the details of sbt usage, see my blog post: [Building Java Projects with sbt](http://xerial.org/blog/2014/03/24/sbt/) ### Building from the source code + See the [build instruction](https://github.com/xerial/snappy-java/blob/master/BUILD.md). Building from the source code is an option when your OS platform and CPU architecture is not supported. To build snappy-java, you need Git, JDK (1.6 or higher), g++ compiler (mingw in Windows) etc. $ git clone https://github.com/xerial/snappy-java.git @@ -157,7 +234,12 @@ When building on Solaris, use `gmake`: A file `target/snappy-java-$(version).jar` is the product additionally containing the native library built for your platform. +### Creating a new release + +GitHub action [https://github.com/xerial/snappy-java/blob/master/.github/workflows/release.yml] will publish a new relase to Maven Central (Sonatype) when a new tag vX.Y.Z is pushed. + ## Miscellaneous Notes + ### Using snappy-java with Tomcat 6 (or higher) Web Server Simply put the snappy-java's jar to WEB-INF/lib folder of your web application. Usual JNI-library specific problem no longer exists since snappy-java version 1.0.3 or higher can be loaded by multiple class loaders. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..02669a53 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 1.1.9.x | :white_check_mark: | +| < 1.1.8.x | :x: | + +## Reporting a Vulnerability + +https://github.com/xerial/snappy-java/security/advisories diff --git a/build.sbt b/build.sbt index 87da4410..6627b970 100644 --- a/build.sbt +++ b/build.sbt @@ -1,66 +1,60 @@ -name := "snappy-java" -organization := "org.xerial.snappy" +Global / onChangedBuildSource := ReloadOnSourceChanges + +name := "snappy-java" +organization := "org.xerial.snappy" organizationName := "xerial.org" -description := "snappy-java: A fast compression/decompression library" -sonatypeProfileName := "org.xerial" +description := "snappy-java: A fast compression/decompression library" -credentials ++= { - if (sys.env.contains("SONATYPE_USERNAME") && sys.env.contains("SONATYPE_PASSWORD")) { - Seq(Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", sys.env("SONATYPE_USERNAME"), sys.env("SONATYPE_PASSWORD"))) - } else { - Seq.empty - } +ThisBuild / publishTo := { + val centralSnapshots = "https://central.sonatype.com/repository/maven-snapshots/" + if (isSnapshot.value) + Some("central-snapshots" at centralSnapshots) + else + localStaging.value } -publishTo := Some( - if (isSnapshot.value) { - Opts.resolver.sonatypeSnapshots +licenses := Seq("Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0.html")) +homepage := Some(url("https://github.com/xerial/snappy-java")) +scmInfo := + Some( + ScmInfo( + browseUrl = url("https://github.com/xerial/snappy-java"), + connection = "scm:git@github.com:xerial/snappy-java.git" + ) + ) + +developers := + List( + Developer( + id = "leo", + name = "Taro L. Saito", + email = "leo@xerial.org", + url = url("http://xerial.org/leo") + ) + ) + +// Use dynamic snapshot version strings for non tagged versions +ThisBuild / dynverSonatypeSnapshots := true +// Use coursier friendly version separator +ThisBuild / dynverSeparator := "-" + +ThisBuild / scalaVersion := "3.7.1" + +// For building jars for JDK8 +ThisBuild / javacOptions ++= { + if (scala.util.Properties.isJavaAtLeast("9")) { + // --release 8 option is not available in JDK8 + Seq("--release", "8") } else { - Opts.resolver.sonatypeStaging + Seq.empty } -) - -pomExtra := { - https://github.com/xerial/snappy-java - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - leo - Taro L. Saito - leo@xerial.org - Xerial Project - - Architect - Project Manager - Chief Developer - - +9 - - - - GitHub - http://github.com/xerial/snappy-java/issues/list - - 2011 - - scm:git@github.com:xerial/snappy-java.git - scm:git:git@github.com:xerial/snappy-java.git - git@github.com:xerial/snappy-java.git - } -scalaVersion in ThisBuild := "2.12.4" - -javacOptions in (Compile, compile) ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation", "-source", "1.7", "-target", "1.7") +Compile / compile / javacOptions ++= + Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation") -javacOptions in doc := { - val opts = Seq("-source", "1.6") +doc / javacOptions := { + val opts = Seq("-source", "1.8") if (scala.util.Properties.isJavaAtLeast("1.8")) opts ++ Seq("-Xdoclint:none") else @@ -68,8 +62,8 @@ javacOptions in doc := { } // Configuration for SnappyHadoopCompatibleOutputStream testing -fork in Test := true -import java.io.File +Test / fork := true + val libTemp = { val path = s"${System.getProperty("java.io.tmpdir")}/snappy_test_${System.currentTimeMillis()}" // certain older Linux systems (debian/trusty in Travis CI) requires the libsnappy.so, loaded by @@ -85,84 +79,74 @@ val linuxLibPath = s"$libTemp:${System.getenv("LD_LIBRARY_PATH")}" // have to add to system dynamic library path since hadoop native library indirectly load libsnappy.1 // can't use javaOptions in Test because it causes the expression to eval twice yielding different temp path values -envVars in Test := Map("XERIAL_SNAPPY_LIB" -> libTemp, "DYLD_LIBRARY_PATH" -> macOSXLibPath, "LD_LIBRARY_PATH" -> linuxLibPath) +Test / envVars := + Map( + "XERIAL_SNAPPY_LIB" -> libTemp, + "DYLD_LIBRARY_PATH" -> macOSXLibPath, + "LD_LIBRARY_PATH" -> linuxLibPath + ) testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v") -concurrentRestrictions in Global := Seq(Tags.limit(Tags.Test, 1)) +Test / parallelExecution := false + autoScalaLibrary := false -crossPaths := false -logBuffered in Test := false - -findbugsReportType := Some(FindbugsReport.FancyHtml) -findbugsReportPath := Some(crossTarget.value / "findbugs" / "report.html") - -libraryDependencies ++= Seq( - "junit" % "junit" % "4.8.2" % "test", - "org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test", - "org.xerial.java" % "xerial-core" % "2.1" % "test", - "org.wvlet.airframe" %% "airframe-log" % "0.25" % "test", - "org.scalatest" %% "scalatest" % "3.0.4" % "test", - "org.osgi" % "org.osgi.core" % "4.3.0" % "provided", - "com.novocode" % "junit-interface" % "0.11" % "test", - "org.apache.hadoop" % "hadoop-common" % "2.7.3" % "test" exclude ("org.xerial.snappy", "snappy-java") -) +crossPaths := false + +libraryDependencies ++= + Seq( + "junit" % "junit" % "4.13.2" % "test", + "org.codehaus.plexus" % "plexus-classworlds" % "2.9.0" % "test", + "org.xerial.java" % "xerial-core" % "2.1" % "test", + "org.wvlet.airframe" %% "airframe-log" % "2025.1.23" % "test", + "org.osgi" % "org.osgi.core" % "6.0.0" % "provided", + "com.github.sbt" % "junit-interface" % "0.13.3" % "test", + "org.apache.hadoop" % "hadoop-common" % "3.4.2" % "test" exclude + ("org.xerial.snappy", "snappy-java") + ) enablePlugins(SbtOsgi) osgiSettings -OsgiKeys.exportPackage := Seq("org.xerial.snappy", "org.xerial.snappy.buffer") +OsgiKeys.exportPackage := + Seq("org.xerial.snappy", "org.xerial.snappy.buffer", "org.xerial.snappy.pool") + OsgiKeys.bundleSymbolicName := "org.xerial.snappy.snappy-java" -OsgiKeys.bundleActivator := Option("org.xerial.snappy.SnappyBundleActivator") -OsgiKeys.importPackage := Seq("""org.osgi.framework;version="[1.5,2)"""") -OsgiKeys.requireCapability := """osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=1.7))"""" - -OsgiKeys.additionalHeaders := Map( - "Bundle-NativeCode" -> Seq( - "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64", - "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64", - "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64", - "org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86", - "org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86", - "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64", - "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64", - "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64", - "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64", - "org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86", - "org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64", - "org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm", - "org/xerial/snappy/native/Linux/armv7/libsnappyjava.so;osname=linux;processor=arm_le", - "org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64le", - "org/xerial/snappy/native/Linux/s390x/libsnappyjava.so;osname=linux;processor=s390x", - "org/xerial/snappy/native/AIX/ppc/libsnappyjava.a;osname=aix;processor=ppc", - "org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64", - "org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86", - "org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64", - "org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc" - ).mkString(","), - "Bundle-DocURL" -> "http://www.xerial.org/", - "Bundle-License" -> "http://www.apache.org/licenses/LICENSE-2.0.txt", - "Bundle-ActivationPolicy" -> "lazy", - "Bundle-Name" -> "snappy-java: A fast compression/decompression library" -) - -import ReleaseTransformations._ -import sbtrelease._ - -releaseTagName := { (version in ThisBuild).value } -releasePublishArtifactsAction := PgpKeys.publishSigned.value - -releaseProcess := Seq[ReleaseStep]( - checkSnapshotDependencies, - inquireVersions, - runClean, - runTest, - setReleaseVersion, - commitReleaseVersion, - tagRelease, - publishArtifacts, - setNextVersion, - commitNextVersion, - releaseStepCommand("sonatypeReleaseAll"), - pushChanges -) +OsgiKeys.bundleActivator := Option("org.xerial.snappy.SnappyBundleActivator") +OsgiKeys.importPackage := Seq("""org.osgi.framework;version="[1.5,2)"""") +OsgiKeys.requireCapability := """osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=1.7))"""" + +OsgiKeys.additionalHeaders := + Map( + "Bundle-NativeCode" -> + Seq( + "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64", + "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64", + "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64", + "org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86", + "org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86", + "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.dylib;osname=macosx;processor=x86-64", + "org/xerial/snappy/native/Mac/aarch64/libsnappyjava.dylib;osname=macosx;processor=aarch64", + "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64", + "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64", + "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64", + "org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86", + "org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64", + "org/xerial/snappy/native/Linux/riscv64/libsnappyjava.so;osname=linux;processor=riscv64", + "org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm", + "org/xerial/snappy/native/Linux/armv7/libsnappyjava.so;osname=linux;processor=arm_le", + "org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64le", + "org/xerial/snappy/native/Linux/s390x/libsnappyjava.so;osname=linux;processor=s390x", + "org/xerial/snappy/native/Linux/loongarch64/libsnappyjava.so;osname=linux;processor=loongarch64", + "org/xerial/snappy/native/Linux/x86_64-musl/libsnappyjava.so;osname=linux;processor=x86-64", + "org/xerial/snappy/native/AIX/ppc/libsnappyjava.a;osname=aix;processor=ppc", + "org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64", + "org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86", + "org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64", + "org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc" + ).mkString(","), + "Bundle-DocURL" -> "http://www.xerial.org/", + "Bundle-License" -> "http://www.apache.org/licenses/LICENSE-2.0.txt", + "Bundle-ActivationPolicy" -> "lazy", + "Bundle-Name" -> "snappy-java: A fast compression/decompression library" + ) diff --git a/docker/Dockerfile.linux-x86_64-musl b/docker/Dockerfile.linux-x86_64-musl new file mode 100644 index 00000000..c76cec95 --- /dev/null +++ b/docker/Dockerfile.linux-x86_64-musl @@ -0,0 +1,36 @@ +FROM alpine:3.18 + +# Set workspace directory +WORKDIR /work + +# Copy the project to the container +COPY . . + +# Install build dependencies +RUN apk add --no-cache \ + openjdk8 \ + cmake \ + make \ + gcc \ + g++ \ + musl-dev \ + linux-headers \ + git \ + util-linux \ + bash \ + curl + +# Install SBT +RUN curl -L "https://github.com/sbt/sbt/releases/download/v1.9.7/sbt-1.9.7.tgz" | tar xz -C /usr/local +ENV PATH="/usr/local/sbt/bin:${PATH}" + +# Install python/pip +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN pip3 install --no-cache --upgrade pip setuptools + +# Set Env Vars +ENV CC=gcc CXX=g++ +ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk +ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/docker/Makefile b/docker/Makefile index f07fd9e1..9648d2b8 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,3 +4,6 @@ centos5-image: # dockerhub login # docker login --username=xerial # docker push xerial/centos5-linux-x86_64:latest + +multiarch-crossbuild-image: + docker build https://github.com/iwasakims/crossbuild.git#fix-osxcross-cmake -t xerial/crossbuild:latest diff --git a/docker/dockcross-aarch64 b/docker/dockcross-aarch64 index 778afb0c..e33291a2 100755 --- a/docker/dockcross-aarch64 +++ b/docker/dockcross-aarch64 @@ -1,6 +1,6 @@ #!/bin/bash -DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64 +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64-lts #------------------------------------------------------------------------------ # Helpers diff --git a/docker/dockcross-android-arm b/docker/dockcross-android-arm index 90990334..43013bc2 100755 --- a/docker/dockcross-android-arm +++ b/docker/dockcross-android-arm @@ -1,16 +1,16 @@ -#!/bin/bash +#!/usr/bin/env bash -DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm +DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm:20230116-670f7f7 #------------------------------------------------------------------------------ # Helpers # err() { - echo -e >&2 ERROR: $@\\n + echo -e >&2 "ERROR: $*\n" } die() { - err $@ + err "$*" exit 1 } @@ -22,28 +22,39 @@ has() { type -t $kind:$name | grep -q function } +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + #------------------------------------------------------------------------------ # Command handlers # command:update-image() { - docker pull $FINAL_IMAGE + $OCI_EXE pull $FINAL_IMAGE } help:update-image() { - echo Pull the latest $FINAL_IMAGE . + echo "Pull the latest $FINAL_IMAGE ." } command:update-script() { - if cmp -s <( docker run $FINAL_IMAGE ) $0; then - echo $0 is up to date + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" else - echo -n Updating $0 '... ' - docker run $FINAL_IMAGE > $0 && echo ok + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok fi } -help:update-image() { - echo Update $0 from $FINAL_IMAGE . +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." } command:update() { @@ -52,7 +63,7 @@ command:update() { } help:update() { - echo Pull the latest $FINAL_IMAGE, and then update $0 from that. + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." } command:help() { @@ -98,6 +109,7 @@ while [[ $# != 0 ]]; do case $1 in --) + shift break ;; @@ -171,29 +183,96 @@ fi # Set the docker run extra args (if any) FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} -# If we are not running via boot2docker -if [ -z $DOCKER_HOST ]; then - USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )" +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; fi #------------------------------------------------------------------------------ # Now, finally, run the command in a container # -docker run --rm \ - -v $PWD:/work \ - $USER_IDS \ +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + --platform linux/amd64 \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ $FINAL_ARGS \ $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code ################################################################################ # # This image is not intended to be run manually. # # To create a dockcross helper script for the -# dockcross/linux-armv7 image, run: +# dockcross/android-arm:20230116-670f7f7 image, run: # -# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7 -# chmod +x dockcross-linux-armv7 +# docker run --rm dockcross/android-arm:20230116-670f7f7 > dockcross-android-arm-20230116-670f7f7 +# chmod +x dockcross-android-arm-20230116-670f7f7 # # You may then wish to move the dockcross script to your PATH. # diff --git a/docker/dockcross-android-arm64 b/docker/dockcross-android-arm64 new file mode 100755 index 00000000..6d4bf85d --- /dev/null +++ b/docker/dockcross-android-arm64 @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm64:20230421-a0eaff4 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/android-arm64:20230421-a0eaff4 image, run: +# +# docker run --rm dockcross/android-arm64:20230421-a0eaff4 > dockcross-android-arm64-20230421-a0eaff4 +# chmod +x dockcross-android-arm64-20230421-a0eaff4 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/docker/dockcross-arm64 b/docker/dockcross-arm64 new file mode 100755 index 00000000..20ffd09a --- /dev/null +++ b/docker/dockcross-arm64 @@ -0,0 +1,242 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64-lts + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 ERROR: $@\\n +} + +die() { + err $@ + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + docker pull $FINAL_IMAGE +} + +help:update-image() { + echo Pull the latest $FINAL_IMAGE . +} + +command:update-script() { + if cmp -s <( docker run --rm $FINAL_IMAGE ) $0; then + echo $0 is up to date + else + echo -n Updating $0 '... ' + docker run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-image() { + echo Update $0 from $FINAL_IMAGE . +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo Pull the latest $FINAL_IMAGE, and then update $0 from that. +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +else + HOST_PWD=$PWD +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" ]; then + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +tty -s && TTY_ARGS=-ti || TTY_ARGS= +CONTAINER_NAME=dockcross_$RANDOM +docker run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/linux-arm64 image, run: +# +# docker run --rm dockcross/linux-arm64 > dockcross-linux-arm64 +# chmod +x dockcross-linux-arm64 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/docker/dockcross-armv6 b/docker/dockcross-armv6 index 2fe16cbb..d68b006e 100755 --- a/docker/dockcross-armv6 +++ b/docker/dockcross-armv6 @@ -1,6 +1,6 @@ #!/bin/bash -DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv6 +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv6-lts #------------------------------------------------------------------------------ # Helpers diff --git a/docker/dockcross-armv7 b/docker/dockcross-armv7 index 756eb49f..49400aec 100755 --- a/docker/dockcross-armv7 +++ b/docker/dockcross-armv7 @@ -1,16 +1,16 @@ -#!/bin/bash +#!/usr/bin/env bash -DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv7 +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv7-lts:20230421-a0eaff4 #------------------------------------------------------------------------------ # Helpers # err() { - echo -e >&2 ERROR: $@\\n + echo -e >&2 "ERROR: $*\n" } die() { - err $@ + err "$*" exit 1 } @@ -22,28 +22,39 @@ has() { type -t $kind:$name | grep -q function } +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + #------------------------------------------------------------------------------ # Command handlers # command:update-image() { - docker pull $FINAL_IMAGE + $OCI_EXE pull $FINAL_IMAGE } help:update-image() { - echo Pull the latest $FINAL_IMAGE . + echo "Pull the latest $FINAL_IMAGE ." } command:update-script() { - if cmp -s <( docker run $FINAL_IMAGE ) $0; then - echo $0 is up to date + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" else - echo -n Updating $0 '... ' - docker run $FINAL_IMAGE > $0 && echo ok + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok fi } -help:update-image() { - echo Update $0 from $FINAL_IMAGE . +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." } command:update() { @@ -52,7 +63,7 @@ command:update() { } help:update() { - echo Pull the latest $FINAL_IMAGE, and then update $0 from that. + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." } command:help() { @@ -98,6 +109,7 @@ while [[ $# != 0 ]]; do case $1 in --) + shift break ;; @@ -171,29 +183,95 @@ fi # Set the docker run extra args (if any) FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} -# If we are not running via boot2docker -if [ -z $DOCKER_HOST ]; then - USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )" +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; fi #------------------------------------------------------------------------------ # Now, finally, run the command in a container # -docker run --rm \ - -v $PWD:/work \ - $USER_IDS \ +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ $FINAL_ARGS \ $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code ################################################################################ # # This image is not intended to be run manually. # # To create a dockcross helper script for the -# dockcross/linux-armv7 image, run: +# dockcross/linux-armv7-lts:20230421-a0eaff4 image, run: # -# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7 -# chmod +x dockcross-linux-armv7 +# docker run --rm dockcross/linux-armv7-lts:20230421-a0eaff4 > dockcross-linux-armv7-lts-20230421-a0eaff4 +# chmod +x dockcross-linux-armv7-lts-20230421-a0eaff4 # # You may then wish to move the dockcross script to your PATH. # diff --git a/docker/dockcross-ppc64le b/docker/dockcross-ppc64le index cf999294..37146f94 100755 --- a/docker/dockcross-ppc64le +++ b/docker/dockcross-ppc64le @@ -1,6 +1,6 @@ #!/bin/bash -DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-ppc64le +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-ppc64le-lts #------------------------------------------------------------------------------ # Helpers diff --git a/docker/dockcross-riscv64 b/docker/dockcross-riscv64 new file mode 100755 index 00000000..3d3779b6 --- /dev/null +++ b/docker/dockcross-riscv64 @@ -0,0 +1,200 @@ +#!/bin/bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-riscv64 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 ERROR: $@\\n +} + +die() { + err $@ + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + docker pull $FINAL_IMAGE +} + +help:update-image() { + echo Pull the latest $FINAL_IMAGE . +} + +command:update-script() { + if cmp -s <( docker run $FINAL_IMAGE ) $0; then + echo $0 is up to date + else + echo -n Updating $0 '... ' + docker run $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-image() { + echo Update $0 from $FINAL_IMAGE . +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo Pull the latest $FINAL_IMAGE, and then update $0 from that. +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# If we are not running via boot2docker +if [ -z $DOCKER_HOST ]; then + USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )" +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +docker run --rm \ + -v $PWD:/work \ + $USER_IDS \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/linux-armv7 image, run: +# +# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7 +# chmod +x dockcross-linux-armv7 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/docker/dockcross-s390x b/docker/dockcross-s390x new file mode 100755 index 00000000..5e1e9fd6 --- /dev/null +++ b/docker/dockcross-s390x @@ -0,0 +1,199 @@ +#!/bin/bash +DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-s390x:20210210-84c47e5 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 ERROR: $@\\n +} + +die() { + err $@ + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + docker pull $FINAL_IMAGE +} + +help:update-image() { + echo Pull the latest $FINAL_IMAGE . +} + +command:update-script() { + if cmp -s <( docker run $FINAL_IMAGE ) $0; then + echo $0 is up to date + else + echo -n Updating $0 '... ' + docker run $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-image() { + echo Update $0 from $FINAL_IMAGE . +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo Pull the latest $FINAL_IMAGE, and then update $0 from that. +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# If we are not running via boot2docker +if [ -z $DOCKER_HOST ]; then + USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )" +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +docker run --rm \ + -v $PWD:/work \ + $USER_IDS \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/linux-armv7 image, run: +# +# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7 +# chmod +x dockcross-linux-armv7 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/docker/dockcross-windows-arm64 b/docker/dockcross-windows-arm64 new file mode 100755 index 00000000..d004745e --- /dev/null +++ b/docker/dockcross-windows-arm64 @@ -0,0 +1,199 @@ +#!/bin/bash +DEFAULT_DOCKCROSS_IMAGE=dockcross/windows-arm64 + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 ERROR: $@\\n +} + +die() { + err $@ + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + docker pull $FINAL_IMAGE +} + +help:update-image() { + echo Pull the latest $FINAL_IMAGE . +} + +command:update-script() { + if cmp -s <( docker run $FINAL_IMAGE ) $0; then + echo $0 is up to date + else + echo -n Updating $0 '... ' + docker run $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-image() { + echo Update $0 from $FINAL_IMAGE . +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo Pull the latest $FINAL_IMAGE, and then update $0 from that. +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# If we are not running via boot2docker +if [ -z $DOCKER_HOST ]; then + USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )" +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +docker run --rm \ + -v $PWD:/work \ + $USER_IDS \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/linux-armv7 image, run: +# +# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7 +# chmod +x dockcross-linux-armv7 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/project/build.properties b/project/build.properties index 71e17047..5bcecde1 100755 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ -sbt.version=1.0.3 +sbt.version=1.12.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index 3bda8300..86eb0b6c 100755 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,4 @@ -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.0") -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0") -addSbtPlugin("com.github.sbt" % "sbt-findbugs" % "2.0.0") -addSbtPlugin("com.github.sbt" % "sbt-jacoco" % "3.0.3") -addSbtPlugin("com.typesafe.sbt" % "sbt-osgi" % "0.9.2") -addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") -addSbtPlugin("com.geirsson" % "sbt-scalafmt" % "1.3.0") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") +addSbtPlugin("com.github.sbt" % "sbt-osgi" % "0.10.0") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.6") +addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.1.1") diff --git a/sbt b/sbt index ffd29c51..5c0c3bde 100755 --- a/sbt +++ b/sbt @@ -2,33 +2,61 @@ # # A more capable sbt runner, coincidentally also called sbt. # Author: Paul Phillips +# https://github.com/paulp/sbt-extras +# +# Generated from http://www.opensource.org/licenses/bsd-license.php +# Copyright (c) 2011, Paul Phillips. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -o pipefail -declare -r sbt_release_version="0.13.16" -declare -r sbt_unreleased_version="0.13.16" +declare -r sbt_release_version="1.9.6" +declare -r sbt_unreleased_version="1.9.6" -declare -r latest_213="2.13.0-M2" -declare -r latest_212="2.12.4" -declare -r latest_211="2.11.11" -declare -r latest_210="2.10.6" +declare -r latest_213="2.13.12" +declare -r latest_212="2.12.18" +declare -r latest_211="2.11.12" +declare -r latest_210="2.10.7" declare -r latest_29="2.9.3" declare -r latest_28="2.8.2" declare -r buildProps="project/build.properties" -declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" +declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases" declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" -declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases" -declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots" +declare -r sbt_launch_mvn_release_repo="https://repo1.maven.org/maven2" +declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots" -declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m" -declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" +declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18" +declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy -Dsbt.coursier.home=project/.coursier" declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new declare sbt_explicit_version declare verbose noshare batch trace_level -declare debugUs declare java_cmd="java" declare sbt_launch_dir="$HOME/.sbt/launchers" @@ -40,13 +68,17 @@ declare -a java_args scalac_args sbt_commands residual_args # args to jvm/sbt via files or environment variables declare -a extra_jvm_opts extra_sbt_opts -echoerr () { echo >&2 "$@"; } -vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } -die () { echo "Aborting: $@" ; exit 1; } +echoerr() { echo >&2 "$@"; } +vlog() { [[ -n "$verbose" ]] && echoerr "$@"; } +die() { + echo "Aborting: $*" + exit 1 +} -setTrapExit () { +setTrapExit() { # save stty and trap exit, to ensure echo is re-enabled if we are interrupted. - export SBT_STTY="$(stty -g 2>/dev/null)" + SBT_STTY="$(stty -g 2>/dev/null)" + export SBT_STTY # restore stty settings (echo in particular) onSbtRunnerExit() { @@ -62,11 +94,14 @@ setTrapExit () { # this seems to cover the bases on OSX, and someone will # have to tell me about the others. -get_script_path () { +get_script_path() { local path="$1" - [[ -L "$path" ]] || { echo "$path" ; return; } + [[ -L "$path" ]] || { + echo "$path" + return + } - local target="$(readlink "$path")" + local -r target="$(readlink "$path")" if [[ "${target:0:1}" == "/" ]]; then echo "$target" else @@ -74,10 +109,12 @@ get_script_path () { fi } -declare -r script_path="$(get_script_path "$BASH_SOURCE")" -declare -r script_name="${script_path##*/}" +script_path="$(get_script_path "${BASH_SOURCE[0]}")" +declare -r script_path +script_name="${script_path##*/}" +declare -r script_name -init_default_option_file () { +init_default_option_file() { local overriding_var="${!1}" local default_file="$2" if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then @@ -89,82 +126,82 @@ init_default_option_file () { echo "$default_file" } -declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" -declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" +sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" +sbtx_opts_file="$(init_default_option_file SBTX_OPTS .sbtxopts)" +jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" -build_props_sbt () { - [[ -r "$buildProps" ]] && \ +build_props_sbt() { + [[ -r "$buildProps" ]] && grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' } -update_build_props_sbt () { - local ver="$1" - local old="$(build_props_sbt)" - - [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && { - perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps" - grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps" - - vlog "!!!" - vlog "!!! Updated file $buildProps setting sbt.version to: $ver" - vlog "!!! Previous value was: $old" - vlog "!!!" - } -} - -set_sbt_version () { +set_sbt_version() { sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version export sbt_version } -url_base () { +url_base() { local version="$1" case "$version" in - 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; - 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;; + 0.7.*) echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;; + 0.10.*) echo "$sbt_launch_ivy_release_repo" ;; 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;; 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_ivy_snapshot_repo" ;; - 0.*) echo "$sbt_launch_ivy_release_repo" ;; - *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_mvn_snapshot_repo" ;; - *) echo "$sbt_launch_mvn_release_repo" ;; + echo "$sbt_launch_ivy_snapshot_repo" ;; + 0.*) echo "$sbt_launch_ivy_release_repo" ;; + *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss" + echo "$sbt_launch_mvn_snapshot_repo" ;; + *) echo "$sbt_launch_mvn_release_repo" ;; esac } -make_url () { +make_url() { local version="$1" local base="${sbt_launch_repo:-$(url_base "$version")}" case "$version" in - 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;; - 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.7.*) echo "$base/sbt-launch-0.7.7.jar" ;; + 0.10.*) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; - 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; - *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;; esac } -addJava () { vlog "[addJava] arg = '$1'" ; java_args+=("$1"); } -addSbt () { vlog "[addSbt] arg = '$1'" ; sbt_commands+=("$1"); } -addScalac () { vlog "[addScalac] arg = '$1'" ; scalac_args+=("$1"); } -addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); } +addJava() { + vlog "[addJava] arg = '$1'" + java_args+=("$1") +} +addSbt() { + vlog "[addSbt] arg = '$1'" + sbt_commands+=("$1") +} +addScalac() { + vlog "[addScalac] arg = '$1'" + scalac_args+=("$1") +} +addResidual() { + vlog "[residual] arg = '$1'" + residual_args+=("$1") +} -addResolver () { addSbt "set resolvers += $1"; } -addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } -setThisBuild () { - vlog "[addBuild] args = '$@'" +addResolver() { addSbt "set resolvers += $1"; } + +addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } + +setThisBuild() { + vlog "[addBuild] args = '$*'" local key="$1" && shift - addSbt "set $key in ThisBuild := $@" + addSbt "set $key in ThisBuild := $*" } -setScalaVersion () { +setScalaVersion() { [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' addSbt "++ $1" } -setJavaHome () { +setJavaHome() { java_cmd="$1/bin/java" setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))" export JAVA_HOME="$1" @@ -172,13 +209,26 @@ setJavaHome () { export PATH="$JAVA_HOME/bin:$PATH" } -getJavaVersion() { "$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \"; } +getJavaVersion() { + local -r str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"') + + # java -version on java8 says 1.8.x + # but on 9 and 10 it's 9.x.y and 10.x.y. + if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then + echo "${BASH_REMATCH[1]}" + # Fixes https://github.com/dwijnand/sbt-extras/issues/326 + elif [[ "$str" =~ ^([0-9]+)(\..*)?(-ea)?$ ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ -n "$str" ]]; then + echoerr "Can't parse java version from: $str" + fi +} checkJava() { # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME - [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" - [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" + [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" + [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" if [[ -n "$java" ]]; then pathJavaVersion=$(getJavaVersion java) @@ -192,31 +242,34 @@ checkJava() { fi } -java_version () { - local version=$(getJavaVersion "$java_cmd") +java_version() { + local -r version=$(getJavaVersion "$java_cmd") vlog "Detected Java version: $version" - echo "${version:2:1}" + echo "$version" } +is_apple_silicon() { [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; } + # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+ -default_jvm_opts () { - local v="$(java_version)" - if [[ $v -ge 8 ]]; then +default_jvm_opts() { + local -r v="$(java_version)" + if [[ $v -ge 17 ]]; then + echo "$default_jvm_opts_common" + elif [[ $v -ge 10 ]]; then + if is_apple_silicon; then + # As of Dec 2020, JVM for Apple Silicon (M1) doesn't support JVMCI + echo "$default_jvm_opts_common" + else + echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler" + fi + elif [[ $v -ge 8 ]]; then echo "$default_jvm_opts_common" else echo "-XX:MaxPermSize=384m $default_jvm_opts_common" fi } -build_props_scala () { - if [[ -r "$buildProps" ]]; then - versionLine="$(grep '^build.scala.versions' "$buildProps")" - versionString="${versionLine##build.scala.versions=}" - echo "${versionString%% .*}" - fi -} - -execRunner () { +execRunner() { # print the arguments one to a line, quoting any containing spaces vlog "# Executing command line:" && { for arg; do @@ -234,40 +287,36 @@ execRunner () { setTrapExit if [[ -n "$batch" ]]; then - "$@" < /dev/null + "$@" /dev/null; then + if command -v curl >/dev/null 2>&1; then curl --fail --silent --location "$url" --output "$jar" - elif which wget >/dev/null; then + elif command -v wget >/dev/null 2>&1; then wget -q -O "$jar" "$url" fi } && [[ -r "$jar" ]] } -acquire_sbt_jar () { +acquire_sbt_jar() { { sbt_jar="$(jar_file "$sbt_version")" [[ -r "$sbt_jar" ]] @@ -276,11 +325,66 @@ acquire_sbt_jar () { [[ -r "$sbt_jar" ]] } || { sbt_jar="$(jar_file "$sbt_version")" - download_url "$(make_url "$sbt_version")" "$sbt_jar" + jar_url="$(make_url "$sbt_version")" + + echoerr "Downloading sbt launcher for ${sbt_version}:" + echoerr " From ${jar_url}" + echoerr " To ${sbt_jar}" + + download_url "${jar_url}" "${sbt_jar}" + + case "${sbt_version}" in + 0.*) + vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check" + echo "" + ;; + *) verify_sbt_jar "${sbt_jar}" ;; + esac } } -usage () { +verify_sbt_jar() { + local jar="${1}" + local md5="${jar}.md5" + md5url="$(make_url "${sbt_version}").md5" + + echoerr "Downloading sbt launcher ${sbt_version} md5 hash:" + echoerr " From ${md5url}" + echoerr " To ${md5}" + + download_url "${md5url}" "${md5}" >/dev/null 2>&1 + + if command -v md5sum >/dev/null 2>&1; then + if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v md5 >/dev/null 2>&1; then + if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v openssl >/dev/null 2>&1; then + if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + else + echoerr "Could not find an MD5 command" + return 1 + fi +} + +usage() { set_sbt_version cat < Run the specified file as a scala script # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) - -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version - -sbt-version use the specified version of sbt (default: $sbt_release_version) - -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version - -sbt-jar use the specified jar as the sbt launcher - -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) - -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) + -sbt-version use the specified version of sbt (default: $sbt_release_version) + -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version + -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version + -sbt-jar use the specified jar as the sbt launcher + -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) + -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) # scala version (default: as chosen by sbt) - -28 use $latest_28 - -29 use $latest_29 - -210 use $latest_210 - -211 use $latest_211 - -212 use $latest_212 - -213 use $latest_213 - -scala-home use the scala build at the specified directory - -scala-version use the specified version of scala - -binary-version use the specified scala version when searching for dependencies + -28 use $latest_28 + -29 use $latest_29 + -210 use $latest_210 + -211 use $latest_211 + -212 use $latest_212 + -213 use $latest_213 + -scala-home use the scala build at the specified directory + -scala-version use the specified version of scala + -binary-version use the specified scala version when searching for dependencies # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) - -java-home alternate JAVA_HOME + -java-home alternate JAVA_HOME # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found - $(default_jvm_opts) - JVM_OPTS environment variable holding either the jvm args directly, or - the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') - Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. - -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) - -Dkey=val pass -Dkey=val directly to the jvm - -J-X pass option -X directly to the jvm (-J is stripped) + $(default_jvm_opts) + JVM_OPTS environment variable holding either the jvm args directly, or + the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') + Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. + -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) + -Dkey=val pass -Dkey=val directly to the jvm + -J-X pass option -X directly to the jvm (-J is stripped) # passing options to sbt, OR to this runner - SBT_OPTS environment variable holding either the sbt args directly, or - the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') - Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. - -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) - -S-X add -X to sbt's scalacOptions (-S is stripped) + SBT_OPTS environment variable holding either the sbt args directly, or + the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') + Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. + -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) + -S-X add -X to sbt's scalacOptions (-S is stripped) + + # passing options exclusively to this runner + SBTX_OPTS environment variable holding either the sbt-extras args directly, or + the reference to a file containing sbt-extras args if given path is prepended by '@' (e.g. '@/etc/sbtxopts') + Note: "@"-file is overridden by local '.sbtxopts' or '-sbtx-opts' argument. + -sbtx-opts file containing sbt-extras args (if not given, .sbtxopts in project root is used if present) EOM + exit 0 } -process_args () { - require_arg () { +process_args() { + require_arg() { local type="$1" local opt="$2" local arg="$3" @@ -367,50 +472,56 @@ process_args () { } while [[ $# -gt 0 ]]; do case "$1" in - -h|-help) usage; exit 0 ;; - -v) verbose=true && shift ;; - -d) addSbt "--debug" && shift ;; - -w) addSbt "--warn" && shift ;; - -q) addSbt "--error" && shift ;; - -x) debugUs=true && shift ;; - -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; - -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; - -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; - -no-share) noshare=true && shift ;; - -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; - -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; - -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; - -offline) addSbt "set offline in Global := true" && shift ;; - -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; - -batch) batch=true && shift ;; - -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; - -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; - - -sbt-create) sbt_create=true && shift ;; - -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -h | -help) usage ;; + -v) verbose=true && shift ;; + -d) addSbt "--debug" && shift ;; + -w) addSbt "--warn" && shift ;; + -q) addSbt "--error" && shift ;; + -x) shift ;; # currently unused + -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; + -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; + + -no-colors) addJava "-Dsbt.log.noformat=true" && addJava "-Dsbt.color=false" && shift ;; + -sbt-create) sbt_create=true && shift ;; + -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; + -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; + -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; + -no-share) noshare=true && shift ;; + -offline) addSbt "set offline in Global := true" && shift ;; + -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; + -batch) batch=true && shift ;; + -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; + -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; + -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; - -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; - -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; - -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; - -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; - -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; - -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; - -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; - -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; - -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; - -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; - - -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; - -S*) addScalac "${1:2}" && shift ;; - -28) setScalaVersion "$latest_28" && shift ;; - -29) setScalaVersion "$latest_29" && shift ;; - -210) setScalaVersion "$latest_210" && shift ;; - -211) setScalaVersion "$latest_211" && shift ;; - -212) setScalaVersion "$latest_212" && shift ;; - -213) setScalaVersion "$latest_213" && shift ;; - new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; - *) addResidual "$1" && shift ;; + -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; + -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; + -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; + -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; + + -28) setScalaVersion "$latest_28" && shift ;; + -29) setScalaVersion "$latest_29" && shift ;; + -210) setScalaVersion "$latest_210" && shift ;; + -211) setScalaVersion "$latest_211" && shift ;; + -212) setScalaVersion "$latest_212" && shift ;; + -213) setScalaVersion "$latest_213" && shift ;; + + -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; + -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; + -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; + -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; + -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; + -sbtx-opts) require_arg path "$1" "$2" && sbtx_opts_file="$2" && shift 2 ;; + -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; + + -D*) addJava "$1" && shift ;; + -J*) addJava "${1:2}" && shift ;; + -S*) addScalac "${1:2}" && shift ;; + + new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; + + *) addResidual "$1" && shift ;; esac done } @@ -422,19 +533,31 @@ process_args "$@" readConfigFile() { local end=false until $end; do - read || end=true + read -r || end=true [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY" - done < "$1" + done <"$1" } # if there are file/environment sbt_opts, process again so we # can supply args to this runner if [[ -r "$sbt_opts_file" ]]; then vlog "Using sbt options defined in file $sbt_opts_file" - while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") + while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then vlog "Using sbt options defined in variable \$SBT_OPTS" - extra_sbt_opts=( $SBT_OPTS ) + IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS" +else + vlog "No extra sbt options have been defined" +fi + +# if there are file/environment sbtx_opts, process again so we +# can supply args to this runner +if [[ -r "$sbtx_opts_file" ]]; then + vlog "Using sbt options defined in file $sbtx_opts_file" + while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbtx_opts_file") +elif [[ -n "$SBTX_OPTS" && ! ("$SBTX_OPTS" =~ ^@.*) ]]; then + vlog "Using sbt options defined in variable \$SBTX_OPTS" + IFS=" " read -r -a extra_sbt_opts <<<"$SBTX_OPTS" else vlog "No extra sbt options have been defined" fi @@ -453,25 +576,24 @@ checkJava # only exists in 0.12+ setTraceLevel() { case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; - *) setThisBuild traceLevel $trace_level ;; + "0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;; + *) setThisBuild traceLevel "$trace_level" ;; esac } # set scalacOptions if we were given any -S opts -[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\"" +[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[*]}\"" -# Update build.properties on disk to set explicit version - sbt gives us no choice -[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && update_build_props_sbt "$sbt_explicit_version" +[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && addJava "-Dsbt.version=$sbt_explicit_version" vlog "Detected sbt version $sbt_version" if [[ -n "$sbt_script" ]]; then - residual_args=( $sbt_script ${residual_args[@]} ) + residual_args=("$sbt_script" "${residual_args[@]}") else # no args - alert them there's stuff in here - (( argumentCount > 0 )) || { + ((argumentCount > 0)) || { vlog "Starting $script_name: invoke with -help for other options" - residual_args=( shell ) + residual_args=(shell) } fi @@ -487,6 +609,7 @@ EOM } # pick up completion if present; todo +# shellcheck disable=SC1091 [[ -r .sbt_completion.sh ]] && source .sbt_completion.sh # directory to store sbt launchers @@ -496,7 +619,7 @@ EOM # no jar? download it. [[ -r "$sbt_jar" ]] || acquire_sbt_jar || { # still no jar? uh-oh. - echo "Download failed. Obtain the jar manually and place it at $sbt_jar" + echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar" exit 1 } @@ -506,12 +629,12 @@ if [[ -n "$noshare" ]]; then done else case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* | "0.12."* ) + "0.7."* | "0.10."* | "0.11."* | "0.12."*) [[ -n "$sbt_dir" ]] || { sbt_dir="$HOME/.sbt/$sbt_version" vlog "Using $sbt_dir as sbt dir, -sbt-dir to override." } - ;; + ;; esac if [[ -n "$sbt_dir" ]]; then @@ -521,58 +644,21 @@ fi if [[ -r "$jvm_opts_file" ]]; then vlog "Using jvm options defined in file $jvm_opts_file" - while read opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") + while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then vlog "Using jvm options defined in \$JVM_OPTS variable" - extra_jvm_opts=( $JVM_OPTS ) + IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS" else vlog "Using default jvm options" - extra_jvm_opts=( $(default_jvm_opts) ) + IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)" fi # traceLevel is 0.12+ [[ -n "$trace_level" ]] && setTraceLevel -main () { - execRunner "$java_cmd" \ - "${extra_jvm_opts[@]}" \ - "${java_args[@]}" \ - -jar "$sbt_jar" \ - "${sbt_commands[@]}" \ - "${residual_args[@]}" -} - -# sbt inserts this string on certain lines when formatting is enabled: -# val OverwriteLine = "\r\u001BM\u001B[2K" -# ...in order not to spam the console with a million "Resolving" lines. -# Unfortunately that makes it that much harder to work with when -# we're not going to print those lines anyway. We strip that bit of -# line noise, but leave the other codes to preserve color. -mainFiltered () { - local ansiOverwrite='\r\x1BM\x1B[2K' - local excludeRegex=$(egrep -v '^#|^$' ~/.sbtignore | paste -sd'|' -) - - echoLine () { - local line="$1" - local line1="$(echo "$line" | sed 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code. - local line2="$(echo "$line1" | sed 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this. - - if [[ $line2 =~ $excludeRegex ]]; then - [[ -n $debugUs ]] && echo "[X] $line1" - else - [[ -n $debugUs ]] && echo " $line1" || echo "$line1" - fi - } - - echoLine "Starting sbt with output filtering enabled." - main | while read -r line; do echoLine "$line"; done -} - -# Only filter if there's a filter file and we don't see a known interactive command. -# Obviously this is super ad hoc but I don't know how to improve on it. Testing whether -# stdin is a terminal is useless because most of my use cases for this filtering are -# exactly when I'm at a terminal, running sbt non-interactively. -shouldFilter () { [[ -f ~/.sbtignore ]] && ! egrep -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } - -# run sbt -if shouldFilter; then mainFiltered; else main; fi +execRunner "$java_cmd" \ + "${extra_jvm_opts[@]}" \ + "${java_args[@]}" \ + -jar "$sbt_jar" \ + "${sbt_commands[@]}" \ + "${residual_args[@]}" diff --git a/script/dynver.sh b/script/dynver.sh new file mode 100755 index 00000000..73f262f9 --- /dev/null +++ b/script/dynver.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# Compute sbt-dynver-compatible version number +BUILD_TIME=`date '+%Y%m%d-%H%M'` +DYN_VER=`git describe --long --tags --abbrev=8 --match "v[0-9]*" --always --dirty="-${BUILD_TIME}"` +GIT_DIST=`echo ${DYN_VER} | sed -re "s/v([^-]*)-([0-9]+)-g(.*)/\2/g"` +GIT_TAG=`git describe --tags --dirty` +RELEASE_VERSION=`echo ${DYN_VER} | sed -re "s/v([^-]*)-([0-9]+)-g(.*)/\1/g"` +SNAPSHOT_VERSION=`echo ${DYN_VER} | sed -re "s/v([^-]*)-([0-9]+)-g(.*)/\1-\2-\3/g"`-SNAPSHOT + +if [ ${GIT_DIST} -eq 0 ]; then + if [ ${GIT_TAG} == *"-dirty" ]; then + VERSION=${SNAPSHOT_VERSION} + else + VERSION=${RELEASE_VERSION} + fi +else + VERSION=${SNAPSHOT_VERSION} +fi + +echo ${VERSION} diff --git a/script/test-jar-integration.sh b/script/test-jar-integration.sh new file mode 100755 index 00000000..af345ad7 --- /dev/null +++ b/script/test-jar-integration.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +echo "==========================================" +echo "Snappy-Java Integration Test" +echo "==========================================" + +# Detect Java version +JAVA_VERSION=$(java -version 2>&1 | head -1 | cut -d'"' -f2 | sed 's/^1\.//' | cut -d'.' -f1) +echo "Java version: $JAVA_VERSION" + +# Build the JAR +echo "" +echo "Building JAR..." +./sbt package + +# Find the JAR +JAR_FILE=$(ls -t target/snappy-java-*.jar | grep -v sources | grep -v javadoc | head -1) +if [ -z "$JAR_FILE" ]; then + echo "ERROR: Could not find snappy-java JAR" + exit 1 +fi +echo "Using JAR: $JAR_FILE" + +# Create temp directory +TEMP_DIR=$(mktemp -d) +echo "" +echo "Using temp directory: $TEMP_DIR" + +# Copy test source +cp src/test/resources/integration/SnappyIntegrationTest.java "$TEMP_DIR/" + +# Compile test +echo "" +echo "Compiling test program..." +javac -cp "$JAR_FILE" -d "$TEMP_DIR" "$TEMP_DIR/SnappyIntegrationTest.java" + +# Run test WITHOUT --enable-native-access flag +echo "" +echo "==========================================" +echo "Running test (WITHOUT --enable-native-access flag)..." +echo "==========================================" + +cd "$TEMP_DIR" +java -cp ".:$OLDPWD/$JAR_FILE" SnappyIntegrationTest 2>&1 +EXIT_CODE=$? +cd - > /dev/null + +echo "" +echo "==========================================" +if [ $EXIT_CODE -eq 0 ]; then + echo "✓ Test PASSED (exit code: $EXIT_CODE)" +else + echo "✗ Test FAILED (exit code: $EXIT_CODE)" +fi +echo "==========================================" + +# Cleanup +rm -rf "$TEMP_DIR" + +exit $EXIT_CODE diff --git a/src/main/java/org/xerial/snappy/BitShuffle.java b/src/main/java/org/xerial/snappy/BitShuffle.java index fa623478..b2a4a6fc 100644 --- a/src/main/java/org/xerial/snappy/BitShuffle.java +++ b/src/main/java/org/xerial/snappy/BitShuffle.java @@ -91,6 +91,9 @@ public static int shuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuf * @throws IOException */ public static byte[] shuffle(short[] input) throws IOException { + if (input.length * 2 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 2]; int numProcessed = impl.shuffle(input, 0, 2, input.length * 2, output, 0); assert(numProcessed == input.length * 2); @@ -105,6 +108,9 @@ public static byte[] shuffle(short[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(int[] input) throws IOException { + if (input.length * 4 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 4]; int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0); assert(numProcessed == input.length * 4); @@ -119,6 +125,9 @@ public static byte[] shuffle(int[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(long[] input) throws IOException { + if (input.length * 8 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 8]; int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0); assert(numProcessed == input.length * 8); @@ -133,6 +142,9 @@ public static byte[] shuffle(long[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(float[] input) throws IOException { + if (input.length * 4 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 4]; int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0); assert(numProcessed == input.length * 4); @@ -147,6 +159,9 @@ public static byte[] shuffle(float[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(double[] input) throws IOException { + if (input.length * 8 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 8]; int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0); assert(numProcessed == input.length * 8); diff --git a/src/main/java/org/xerial/snappy/OSInfo.java b/src/main/java/org/xerial/snappy/OSInfo.java index d8b52a71..c34a5fc3 100755 --- a/src/main/java/org/xerial/snappy/OSInfo.java +++ b/src/main/java/org/xerial/snappy/OSInfo.java @@ -25,6 +25,7 @@ package org.xerial.snappy; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; @@ -44,9 +45,12 @@ public class OSInfo { public static final String IA64 = "ia64"; public static final String PPC = "ppc"; public static final String PPC64 = "ppc64"; + public static final String PPC64LE = "ppc64le"; public static final String IBMZ = "s390"; public static final String IBMZ_64 = "s390x"; public static final String AARCH_64 = "aarch64"; + public static final String RISCV_64 = "riscv64"; + public static final String LOONGARCH_64 = "loongarch64"; static { // x86 mappings @@ -78,6 +82,9 @@ public class OSInfo { archMapping.put("power_pc", PPC); archMapping.put("power_rs", PPC); + //PowerPC 64bit Little Endian mappings + archMapping.put(PPC64LE, PPC64LE); + // TODO: PowerPC 64bit mappings archMapping.put(PPC64, PPC64); archMapping.put("power64", PPC64); @@ -93,6 +100,12 @@ public class OSInfo { // Aarch64 mappings archMapping.put(AARCH_64, AARCH_64); + + // RISC-V mappings + archMapping.put(RISCV_64, RISCV_64); + + // LoongArch64 mappings + archMapping.put(LOONGARCH_64, LOONGARCH_64); } public static void main(String[] args) { @@ -164,7 +177,7 @@ else if(armType.startsWith("armv7")) { } // Java 1.8 introduces a system property to determine armel or armhf - // http://bugs.java.com/bugdatabase/view_bug.do?bug_id=8005545 + // https://bugs.openjdk.org/browse/JDK-8005545 String abi = System.getProperty("sun.arch.abi"); if(abi != null && abi.startsWith("gnueabihf")) { return "armv7"; @@ -207,6 +220,10 @@ public static String getArchName() { return "android-arm"; } + if (isX64Musl()) { + return "x86_64-musl"; + } + if(osArch.startsWith("arm")) { osArch = resolveArmArchType(); } @@ -218,6 +235,14 @@ public static String getArchName() { return translateArchNameToFolderName(osArch); } + private static boolean isX64Musl() { + try { + return new File("/lib/ld-musl-x86_64.so.1").exists(); + } catch (SecurityException e) { + return false; + } + } + static String translateOSNameToFolderName(String osName) { if(osName.contains("Windows")) { return "Windows"; diff --git a/src/main/java/org/xerial/snappy/PureJavaCrc32C.java b/src/main/java/org/xerial/snappy/PureJavaCrc32C.java index bed1f767..5991101a 100644 --- a/src/main/java/org/xerial/snappy/PureJavaCrc32C.java +++ b/src/main/java/org/xerial/snappy/PureJavaCrc32C.java @@ -48,8 +48,7 @@ public int getIntegerValue() /** {@inheritDoc} */ public long getValue() { - long ret = crc; - return (~ret) & 0xffffffffL; + return (~crc) & 0xffffffffL; } /** {@inheritDoc} */ diff --git a/src/main/java/org/xerial/snappy/Snappy.java b/src/main/java/org/xerial/snappy/Snappy.java index 31d0d2ee..14d17964 100755 --- a/src/main/java/org/xerial/snappy/Snappy.java +++ b/src/main/java/org/xerial/snappy/Snappy.java @@ -19,7 +19,7 @@ // Snappy.java // Since: 2011/03/29 // -// $URL$ +// $URL$ // $Author$ //-------------------------------------- package org.xerial.snappy; @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URL; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.Properties; @@ -43,18 +44,13 @@ public class Snappy { static { - try { - impl = SnappyLoader.loadSnappyApi(); - } - catch (Exception e) { - throw new ExceptionInInitializerError(e); - } + init(); } /** * An instance of SnappyNative */ - private static SnappyNative impl; + private static SnappyApi impl; /** * Clean up a temporary file (native lib) generated by snappy-java. @@ -69,6 +65,15 @@ public static void cleanUp() SnappyLoader.setSnappyApi(null); } + static void init() { + try { + impl = SnappyLoader.loadSnappyApi(); + } + catch (Exception e) { + throw new ExceptionInInitializerError(e); + } + } + /** * Copy bytes from source to destination * @@ -144,12 +149,13 @@ public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) // output: compressed int uPos = uncompressed.position(); int uLen = uncompressed.remaining(); + int cPos = compressed.position(); int compressedSize = impl.rawCompress(uncompressed, uPos, uLen, compressed, - compressed.position()); + cPos); // pos limit // [ ......BBBBBBB.........] - compressed.limit(compressed.position() + compressedSize); + ((Buffer) compressed).limit(cPos + compressedSize); return compressedSize; } @@ -163,7 +169,11 @@ public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) public static byte[] compress(char[] input) throws IOException { - return rawCompress(input, input.length * 2); // char uses 2 bytes + int byteSize = input.length * 2; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // char uses 2 bytes } /** @@ -175,7 +185,11 @@ public static byte[] compress(char[] input) public static byte[] compress(double[] input) throws IOException { - return rawCompress(input, input.length * 8); // double uses 8 bytes + int byteSize = input.length * 8; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // double uses 8 bytes } /** @@ -187,7 +201,11 @@ public static byte[] compress(double[] input) public static byte[] compress(float[] input) throws IOException { - return rawCompress(input, input.length * 4); // float uses 4 bytes + int byteSize = input.length * 4; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // float uses 4 bytes } /** @@ -199,7 +217,11 @@ public static byte[] compress(float[] input) public static byte[] compress(int[] input) throws IOException { - return rawCompress(input, input.length * 4); // int uses 4 bytes + int byteSize = input.length * 4; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // int uses 4 bytes } /** @@ -211,7 +233,11 @@ public static byte[] compress(int[] input) public static byte[] compress(long[] input) throws IOException { - return rawCompress(input, input.length * 8); // long uses 8 bytes + int byteSize = input.length * 8; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // long uses 8 bytes } /** @@ -223,7 +249,11 @@ public static byte[] compress(long[] input) public static byte[] compress(short[] input) throws IOException { - return rawCompress(input, input.length * 2); // short uses 2 bytes + int byteSize = input.length * 2; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // short uses 2 bytes } /** @@ -541,12 +571,13 @@ public static int uncompress(ByteBuffer compressed, ByteBuffer uncompressed) int cPos = compressed.position(); int cLen = compressed.remaining(); + int uPos = uncompressed.position(); // pos limit // [ ......UUUUUU.........] int decompressedSize = impl.rawUncompress(compressed, cPos, cLen, uncompressed, - uncompressed.position()); - uncompressed.limit(uncompressed.position() + decompressedSize); + uPos); + uncompressed.limit(uPos + decompressedSize); return decompressedSize; } @@ -592,9 +623,24 @@ public static char[] uncompressCharArray(byte[] input, int offset, int length) public static double[] uncompressDoubleArray(byte[] input) throws IOException { - int uncompressedLength = Snappy.uncompressedLength(input, 0, input.length); + return uncompressDoubleArray(input, 0, input.length); + } + + /** + * Uncompress the input as a double array + * + * @param input + * @param offset + * @param length + * @return the uncompressed data + * @throws IOException + */ + public static double[] uncompressDoubleArray(byte[] input, int offset, int length) + throws IOException + { + int uncompressedLength = Snappy.uncompressedLength(input, offset, length); double[] result = new double[uncompressedLength / 8]; - impl.rawUncompress(input, 0, input.length, result, 0); + impl.rawUncompress(input, offset, length, result, 0); return result; } diff --git a/src/main/java/org/xerial/snappy/SnappyApi.java b/src/main/java/org/xerial/snappy/SnappyApi.java new file mode 100644 index 00000000..ab7f28a6 --- /dev/null +++ b/src/main/java/org/xerial/snappy/SnappyApi.java @@ -0,0 +1,60 @@ +package org.xerial.snappy; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Snappy compressor/decompressor interface. The implementation can be JNI binding or pure-java Snappy implementation. + */ +public interface SnappyApi +{ + // ------------------------------------------------------------------------ + // Generic compression/decompression routines. + // ------------------------------------------------------------------------ + long rawCompress(long inputAddr, long inputSize, long destAddr) + throws IOException; + + long rawUncompress(long inputAddr, long inputSize, long destAddr) + throws IOException; + + int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, + int outputOffset) + throws IOException; + + int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) + throws IOException; + + int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed, + int outputOffset) + throws IOException; + + int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset) + throws IOException; + + // Returns the maximal size of the compressed representation of + // input data that is "source_bytes" bytes in length; + int maxCompressedLength(int source_bytes); + + // This operation takes O(1) time. + int uncompressedLength(ByteBuffer compressed, int offset, int len) + throws IOException; + + int uncompressedLength(Object input, int offset, int len) + throws IOException; + + long uncompressedLength(long inputAddr, long len) + throws IOException; + + boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) + throws IOException; + + boolean isValidCompressedBuffer(Object input, int offset, int len) + throws IOException; + + boolean isValidCompressedBuffer(long inputAddr, long offset, long len) + throws IOException; + + void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) + throws IOException; + +} diff --git a/src/main/java/org/xerial/snappy/SnappyBundleActivator.java b/src/main/java/org/xerial/snappy/SnappyBundleActivator.java index dec838c1..cd9d0ab7 100755 --- a/src/main/java/org/xerial/snappy/SnappyBundleActivator.java +++ b/src/main/java/org/xerial/snappy/SnappyBundleActivator.java @@ -51,7 +51,9 @@ public void start(BundleContext context) throws Exception { String library = System.mapLibraryName(LIBRARY_NAME); - if (library.toLowerCase().endsWith(".dylib")) { + String osArch = System.getProperty("os.arch"); + + if (library.toLowerCase().endsWith(".dylib") && "x86".equals(osArch)) { // some MacOS JDK7+ vendors map to dylib instead of jnilib library = library.replace(".dylib", ".jnilib"); } diff --git a/src/main/java/org/xerial/snappy/SnappyErrorCode.java b/src/main/java/org/xerial/snappy/SnappyErrorCode.java index 4325b02e..661ffd84 100755 --- a/src/main/java/org/xerial/snappy/SnappyErrorCode.java +++ b/src/main/java/org/xerial/snappy/SnappyErrorCode.java @@ -41,7 +41,9 @@ public enum SnappyErrorCode FAILED_TO_UNCOMPRESS(5), EMPTY_INPUT(6), INCOMPATIBLE_VERSION(7), - INVALID_CHUNK_SIZE(8); + INVALID_CHUNK_SIZE(8), + UNSUPPORTED_PLATFORM(9), + TOO_LARGE_INPUT(10); public final int id; diff --git a/src/main/java/org/xerial/snappy/SnappyFramed.java b/src/main/java/org/xerial/snappy/SnappyFramed.java index 0fef08b2..0c28eac2 100644 --- a/src/main/java/org/xerial/snappy/SnappyFramed.java +++ b/src/main/java/org/xerial/snappy/SnappyFramed.java @@ -1,187 +1,172 @@ -/* - * Created: Apr 12, 2013 - */ -package org.xerial.snappy; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Constants and utilities for implementing x-snappy-framed. - * - * @author Brett Okken - * @since 1.1.0 - */ -final class SnappyFramed -{ - public static final int COMPRESSED_DATA_FLAG = 0x00; - - public static final int UNCOMPRESSED_DATA_FLAG = 0x01; - - public static final int STREAM_IDENTIFIER_FLAG = 0xff; - - private static final int MASK_DELTA = 0xa282ead8; - - /** - * Sun specific mechanisms to clean up resources associated with direct byte buffers. - */ - @SuppressWarnings("unchecked") - private static final Class SUN_DIRECT_BUFFER = (Class) lookupClassQuietly("sun.nio.ch.DirectBuffer"); - private static final Method SUN_BUFFER_CLEANER; - private static final Method SUN_CLEANER_CLEAN; - - static { - Method bufferCleaner = null; - Method cleanerClean = null; - try { - //operate under the assumption that if the sun direct buffer class exists, - //all of the sun classes exist - if (SUN_DIRECT_BUFFER != null) { - bufferCleaner = SUN_DIRECT_BUFFER.getMethod("cleaner", (Class[]) null); - Class cleanClazz = lookupClassQuietly("sun.misc.Cleaner"); - cleanerClean = cleanClazz.getMethod("clean", (Class[]) null); - } - } - catch (Throwable t) { - Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t); - } - SUN_BUFFER_CLEANER = bufferCleaner; - SUN_CLEANER_CLEAN = cleanerClean; - } - - /** - * The header consists of the stream identifier flag, 3 bytes indicating a - * length of 6, and "sNaPpY" in ASCII. - */ - public static final byte[] HEADER_BYTES = new byte[] { - (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, - 0x50, 0x70, 0x59}; - - public static int maskedCrc32c(byte[] data) - { - return maskedCrc32c(data, 0, data.length); - } - - public static int maskedCrc32c(byte[] data, int offset, int length) - { - final PureJavaCrc32C crc32c = new PureJavaCrc32C(); - crc32c.update(data, offset, length); - return mask(crc32c.getIntegerValue()); - } - - /** - * Checksums are not stored directly, but masked, as checksumming data and - * then its own checksum can be problematic. The masking is the same as used - * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant - * 0xa282ead8 (using wraparound as normal for unsigned integers). This is - * equivalent to the following C code: - *

- * 

-     * uint32_t mask_checksum(uint32_t x) {
-     *     return ((x >> 15) | (x << 17)) + 0xa282ead8;
-     * }
-     *
- */ - public static int mask(int crc) - { - // Rotate right by 15 bits and add a constant. - return ((crc >>> 15) | (crc << 17)) + MASK_DELTA; - } - - static final int readBytes(ReadableByteChannel source, ByteBuffer dest) - throws IOException - { - // tells how many bytes to read. - final int expectedLength = dest.remaining(); - - int totalRead = 0; - - // how many bytes were read. - int lastRead = source.read(dest); - - totalRead = lastRead; - - // if we did not read as many bytes as we had hoped, try reading again. - if (lastRead < expectedLength) { - // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading. - while (dest.remaining() != 0 && lastRead != -1) { - lastRead = source.read(dest); - - // if we got EOF, do not add to total read. - if (lastRead != -1) { - totalRead += lastRead; - } - } - } - - if (totalRead > 0) { - dest.limit(dest.position()); - } - else { - dest.position(dest.limit()); - } - - return totalRead; - } - - static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) - throws IOException - { - if (skip <= 0) { - return 0; - } - - int toSkip = skip; - int skipped = 0; - while (toSkip > 0 && skipped != -1) { - buffer.clear(); - if (toSkip < buffer.capacity()) { - buffer.limit(toSkip); - } - - skipped = source.read(buffer); - if (skipped > 0) { - toSkip -= skipped; - } - } - - buffer.clear(); - return skip - toSkip; - } - - private static Class lookupClassQuietly(String name) - { - try { - return SnappyFramed.class.getClassLoader().loadClass(name); - } - catch (Throwable t) { - Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t); - } - - return null; - } - - /** - * Provides jvm implementation specific operation to aggressively release resources associated with buffer. - * - * @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}. - */ - static void releaseDirectByteBuffer(ByteBuffer buffer) - { - assert buffer != null && buffer.isDirect(); - - if (SUN_DIRECT_BUFFER != null && SUN_DIRECT_BUFFER.isAssignableFrom(buffer.getClass())) { - try { - Object cleaner = SUN_BUFFER_CLEANER.invoke(buffer, (Object[]) null); - SUN_CLEANER_CLEAN.invoke(cleaner, (Object[]) null); - } - catch (Throwable t) { - Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t); - } - } - } -} +/* + * Created: Apr 12, 2013 + */ +package org.xerial.snappy; + +import java.io.IOException; +import java.lang.invoke.LambdaMetafactory; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.Checksum; + +/** + * Constants and utilities for implementing x-snappy-framed. + * + * @author Brett Okken + * @since 1.1.0 + */ +final class SnappyFramed +{ + public static final int COMPRESSED_DATA_FLAG = 0x00; + + public static final int UNCOMPRESSED_DATA_FLAG = 0x01; + + public static final int STREAM_IDENTIFIER_FLAG = 0xff; + + private static final int MASK_DELTA = 0xa282ead8; + + private static final Supplier CHECKSUM_SUPPLIER; + + static + { + Supplier supplier = null; + try + { + final Class crc32cClazz = Class.forName("java.util.zip.CRC32C"); + // using LambdaMetafactory requires a caller sensitive lookup + final MethodHandles.Lookup lookup = MethodHandles.lookup(); + final MethodHandle conHandle = lookup.findConstructor(crc32cClazz, MethodType.methodType(void.class)); + + // use LambdaMetafactory to generate an implementation of Supplier which invokes + // the java.util.zip.CRC32C default constructor + supplier = (Supplier) LambdaMetafactory.metafactory(lookup, + // method name on Supplier + "get", + // functional interface to be created by factory + MethodType.methodType(Supplier.class), + // type of the functional interface + // uses a generic, so erasure to Object + MethodType.methodType(Object.class), + // the method handle to call + conHandle, + // type as used at call site + MethodType.methodType(Checksum.class)) + .getTarget() + .invoke(); + } + catch(Throwable t) + { + Logger.getLogger(SnappyFramed.class.getName()) + .log(Level.FINE, "java.util.zip.CRC32C not loaded, using PureJavaCrc32C", t); + supplier = null; + } + + CHECKSUM_SUPPLIER = supplier != null ? supplier : PureJavaCrc32C::new; + } + + /** + * The header consists of the stream identifier flag, 3 bytes indicating a + * length of 6, and "sNaPpY" in ASCII. + */ + public static final byte[] HEADER_BYTES = new byte[] { + (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, + 0x50, 0x70, 0x59}; + + public static Checksum getCRC32C() + { + return CHECKSUM_SUPPLIER.get(); + } + + public static int maskedCrc32c(Checksum crc32c, byte[] data, int offset, int length) + { + crc32c.reset(); + crc32c.update(data, offset, length); + return mask((int) crc32c.getValue()); + } + + /** + * Checksums are not stored directly, but masked, as checksumming data and + * then its own checksum can be problematic. The masking is the same as used + * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant + * 0xa282ead8 (using wraparound as normal for unsigned integers). This is + * equivalent to the following C code: + *

+ * 

+     * uint32_t mask_checksum(uint32_t x) {
+     *     return ((x >> 15) | (x << 17)) + 0xa282ead8;
+     * }
+     *
+ */ + public static int mask(int crc) + { + // Rotate right by 15 bits and add a constant. + return ((crc >>> 15) | (crc << 17)) + MASK_DELTA; + } + + static final int readBytes(ReadableByteChannel source, ByteBuffer dest) + throws IOException + { + // tells how many bytes to read. + final int expectedLength = dest.remaining(); + + int totalRead = 0; + + // how many bytes were read. + int lastRead = source.read(dest); + + totalRead = lastRead; + + // if we did not read as many bytes as we had hoped, try reading again. + if (lastRead < expectedLength) { + // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading. + while (dest.remaining() != 0 && lastRead != -1) { + lastRead = source.read(dest); + + // if we got EOF, do not add to total read. + if (lastRead != -1) { + totalRead += lastRead; + } + } + } + + if (totalRead > 0) { + dest.limit(dest.position()); + } + else { + dest.position(dest.limit()); + } + + return totalRead; + } + + static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) + throws IOException + { + if (skip <= 0) { + return 0; + } + + int toSkip = skip; + int skipped = 0; + while (toSkip > 0 && skipped != -1) { + buffer.clear(); + if (toSkip < buffer.capacity()) { + buffer.limit(toSkip); + } + + skipped = source.read(buffer); + if (skipped > 0) { + toSkip -= skipped; + } + } + + buffer.clear(); + return skip - toSkip; + } +} diff --git a/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java b/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java index 21eb30c8..8e975735 100644 --- a/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java @@ -1,588 +1,678 @@ -/* - * Created: Apr 15, 2013 - */ -package org.xerial.snappy; - -import static java.lang.Math.min; -import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; -import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; -import static org.xerial.snappy.SnappyFramed.STREAM_IDENTIFIER_FLAG; -import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; -import static org.xerial.snappy.SnappyFramed.readBytes; -import static org.xerial.snappy.SnappyFramed.releaseDirectByteBuffer; -import static org.xerial.snappy.SnappyFramedOutputStream.MAX_BLOCK_SIZE; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; -import java.util.Arrays; - -/** - * Implements the x-snappy-framed as an {@link InputStream} and - * {@link ReadableByteChannel}. - * - * @author Brett Okken - * @since 1.1.0 - */ -public final class SnappyFramedInputStream - extends InputStream - implements - ReadableByteChannel -{ - - private final ReadableByteChannel rbc; - private final ByteBuffer frameHeader; - private final boolean verifyChecksums; - - /** - * A single frame read from the underlying {@link InputStream}. - */ - private ByteBuffer input; - - /** - * The decompressed data from {@link #input}. - */ - private ByteBuffer uncompressedDirect; - - /** - * Indicates if this instance has been closed. - */ - private boolean closed; - - /** - * Indicates if we have reached the EOF on {@link #input}. - */ - private boolean eof; - - /** - * The position in {@link #input} buffer to read to. - */ - private int valid; - - /** - * The next position to read from {@link #buffer}. - */ - private int position; - - /** - * Buffer contains a copy of the uncompressed data for the block. - */ - private byte[] buffer; - - /** - * Creates a Snappy input stream to read data from the specified underlying - * input stream. - * - * @param in the underlying input stream. Must not be {@code null}. - */ - public SnappyFramedInputStream(InputStream in) - throws IOException - { - this(in, true); - } - - /** - * Creates a Snappy input stream to read data from the specified underlying - * input stream. - * - * @param in the underlying input stream. Must not be {@code null}. - * @param verifyChecksums if true, checksums in input stream will be verified - */ - public SnappyFramedInputStream(InputStream in, boolean verifyChecksums) - throws IOException - { - this(Channels.newChannel(in), verifyChecksums); - } - - /** - * Creates a Snappy input stream to read data from the specified underlying - * channel. - * - * @param in the underlying readable channel. Must not be {@code null}. - */ - public SnappyFramedInputStream(ReadableByteChannel in) - throws IOException - { - this(in, true); - } - - /** - * Creates a Snappy input stream to read data from the specified underlying - * channel. - * - * @param in the underlying readable channel. Must not be {@code null}. - * @param verifyChecksums if true, checksums in input stream will be verified - */ - public SnappyFramedInputStream(ReadableByteChannel in, - boolean verifyChecksums) - throws IOException - { - if (in == null) { - throw new NullPointerException("in is null"); - } - - this.rbc = in; - this.verifyChecksums = verifyChecksums; - - allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5); - this.frameHeader = ByteBuffer.allocate(4); - - // stream must begin with stream header - final byte[] expectedHeader = HEADER_BYTES; - final byte[] actualHeader = new byte[expectedHeader.length]; - final ByteBuffer actualBuffer = ByteBuffer.wrap(actualHeader); - - final int read = SnappyFramed.readBytes(in, actualBuffer); - if (read < expectedHeader.length) { - throw new EOFException( - "encountered EOF while reading stream header"); - } - if (!Arrays.equals(expectedHeader, actualHeader)) { - throw new IOException("invalid stream header"); - } - } - - /** - * @param size - */ - private void allocateBuffersBasedOnSize(int size) - { - - if (input != null) { - releaseDirectByteBuffer(input); - } - - if (uncompressedDirect != null) { - releaseDirectByteBuffer(uncompressedDirect); - } - - input = ByteBuffer.allocateDirect(size); - final int maxCompressedLength = Snappy.maxCompressedLength(size); - uncompressedDirect = ByteBuffer.allocateDirect(maxCompressedLength); - buffer = new byte[maxCompressedLength]; - } - - @Override - public int read() - throws IOException - { - if (closed) { - return -1; - } - if (!ensureBuffer()) { - return -1; - } - return buffer[position++] & 0xFF; - } - - @Override - public int read(byte[] output, int offset, int length) - throws IOException - { - - if (output == null) { - throw new IllegalArgumentException("output is null"); - } - - if (offset < 0 || length < 0 || offset + length > output.length) { - throw new IllegalArgumentException("invalid offset [" + offset - + "] and length [" + length + ']'); - } - - if (closed) { - throw new ClosedChannelException(); - } - - if (length == 0) { - return 0; - } - if (!ensureBuffer()) { - return -1; - } - - final int size = min(length, available()); - System.arraycopy(buffer, position, output, offset, size); - position += size; - return size; - } - - @Override - public int available() - throws IOException - { - if (closed) { - return 0; - } - return valid - position; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isOpen() - { - return !closed; - } - - /** - * {@inheritDoc} - */ - @Override - public int read(ByteBuffer dst) - throws IOException - { - - if (dst == null) { - throw new IllegalArgumentException("dst is null"); - } - - if (closed) { - throw new ClosedChannelException(); - } - - if (dst.remaining() == 0) { - return 0; - } - if (!ensureBuffer()) { - return -1; - } - - final int size = min(dst.remaining(), available()); - dst.put(buffer, position, size); - position += size; - return size; - } - - /** - * Transfers the entire content of this {@link InputStream} to os. - * This potentially limits the amount of buffering required to decompress - * content. - *

- * Unlike {@link #read(byte[], int, int)}, this method does not need to be - * called multiple times. A single call will transfer all available content. - * Any calls after the source has been exhausted will result in a return - * value of {@code 0}. - *

- * - * @param os The destination to write decompressed content to. - * @return The number of bytes transferred. - * @throws IOException - * @since 1.1.1 - */ - public long transferTo(OutputStream os) - throws IOException - { - if (os == null) { - throw new IllegalArgumentException("os is null"); - } - - if (closed) { - throw new ClosedChannelException(); - } - - long totTransfered = 0; - - while (ensureBuffer()) { - final int available = available(); - os.write(buffer, position, available); - position += available; - totTransfered += available; - } - - return totTransfered; - } - - /** - * Transfers the entire content of this {@link ReadableByteChannel} to - * wbc. This potentially limits the amount of buffering required to - * decompress content. - *

- *

- * Unlike {@link #read(ByteBuffer)}, this method does not need to be called - * multiple times. A single call will transfer all available content. Any - * calls after the source has been exhausted will result in a return value - * of {@code 0}. - *

- * - * @param wbc The destination to write decompressed content to. - * @return The number of bytes transferred. - * @throws IOException - * @since 1.1.1 - */ - public long transferTo(WritableByteChannel wbc) - throws IOException - { - if (wbc == null) { - throw new IllegalArgumentException("wbc is null"); - } - - if (closed) { - throw new ClosedChannelException(); - } - - final ByteBuffer bb = ByteBuffer.wrap(buffer); - - long totTransfered = 0; - - while (ensureBuffer()) { - bb.clear(); - bb.position(position); - bb.limit(position + available()); - - wbc.write(bb); - - final int written = bb.position() - position; - position += written; - - totTransfered += written; - } - - return totTransfered; - } - - @Override - public void close() - throws IOException - { - try { - rbc.close(); - } - finally { - if (!closed) { - closed = true; - } - - if (input != null) { - releaseDirectByteBuffer(input); - } - - if (uncompressedDirect != null) { - releaseDirectByteBuffer(uncompressedDirect); - } - } - } - - static enum FrameAction - { - RAW, SKIP, UNCOMPRESS; - } - - public static final class FrameMetaData - { - final int length; - final FrameAction frameAction; - - /** - * @param frameAction - * @param length - */ - public FrameMetaData(FrameAction frameAction, int length) - { - super(); - this.frameAction = frameAction; - this.length = length; - } - } - - public static final class FrameData - { - final int checkSum; - final int offset; - - /** - * @param checkSum - * @param offset - */ - public FrameData(int checkSum, int offset) - { - super(); - this.checkSum = checkSum; - this.offset = offset; - } - } - - private boolean ensureBuffer() - throws IOException - { - if (available() > 0) { - return true; - } - if (eof) { - return false; - } - - if (!readBlockHeader()) { - eof = true; - return false; - } - - // get action based on header - final FrameMetaData frameMetaData = getFrameMetaData(frameHeader); - - if (FrameAction.SKIP == frameMetaData.frameAction) { - SnappyFramed.skip(rbc, frameMetaData.length, - ByteBuffer.wrap(buffer)); - return ensureBuffer(); - } - - if (frameMetaData.length > input.capacity()) { - allocateBuffersBasedOnSize(frameMetaData.length); - } - - input.clear(); - input.limit(frameMetaData.length); - - final int actualRead = readBytes(rbc, input); - if (actualRead != frameMetaData.length) { - throw new EOFException("unexpectd EOF when reading frame"); - } - input.flip(); - - final FrameData frameData = getFrameData(input); - - if (FrameAction.UNCOMPRESS == frameMetaData.frameAction) { - - input.position(frameData.offset); - - final int uncompressedLength = Snappy.uncompressedLength(input); - - if (uncompressedLength > uncompressedDirect.capacity()) { - uncompressedDirect = ByteBuffer - .allocateDirect(uncompressedLength); - buffer = new byte[Math.max(input.capacity(), uncompressedLength)]; - } - - uncompressedDirect.clear(); - - this.valid = Snappy.uncompress(input, uncompressedDirect); - - uncompressedDirect.get(buffer, 0, valid); - this.position = 0; - } - else { - // we need to start reading at the offset - input.position(frameData.offset); - this.position = 0; - this.valid = input.remaining(); - this.input.get(buffer, 0, input.remaining()); - } - - if (verifyChecksums) { - final int actualCrc32c = SnappyFramed.maskedCrc32c(buffer, - position, valid - position); - if (frameData.checkSum != actualCrc32c) { - throw new IOException("Corrupt input: invalid checksum"); - } - } - - return true; - } - - private boolean readBlockHeader() - throws IOException - { - frameHeader.clear(); - int read = readBytes(rbc, frameHeader); - - if (read == -1) { - return false; - } - - if (read < frameHeader.capacity()) { - throw new EOFException("encountered EOF while reading block header"); - } - frameHeader.flip(); - - return true; - } - - /** - * @param frameHeader - * @return - * @throws IOException - */ - private FrameMetaData getFrameMetaData(ByteBuffer frameHeader) - throws IOException - { - - assert frameHeader.hasArray(); - - final byte[] frameHeaderArray = frameHeader.array(); - - int length = (frameHeaderArray[1] & 0xFF); - length |= (frameHeaderArray[2] & 0xFF) << 8; - length |= (frameHeaderArray[3] & 0xFF) << 16; - - int minLength = 0; - final FrameAction frameAction; - final int flag = frameHeaderArray[0] & 0xFF; - switch (flag) { - case COMPRESSED_DATA_FLAG: - frameAction = FrameAction.UNCOMPRESS; - minLength = 5; - break; - case UNCOMPRESSED_DATA_FLAG: - frameAction = FrameAction.RAW; - minLength = 5; - break; - case STREAM_IDENTIFIER_FLAG: - if (length != 6) { - throw new IOException( - "stream identifier chunk with invalid length: " - + length); - } - frameAction = FrameAction.SKIP; - minLength = 6; - break; - default: - // Reserved unskippable chunks (chunk types 0x02-0x7f) - if (flag <= 0x7f) { - throw new IOException("unsupported unskippable chunk: " - + Integer.toHexString(flag)); - } - - // all that is left is Reserved skippable chunks (chunk types - // 0x80-0xfe) - frameAction = FrameAction.SKIP; - minLength = 0; - } - - if (length < minLength) { - throw new IOException("invalid length: " + length - + " for chunk flag: " + Integer.toHexString(flag)); - } - - return new FrameMetaData(frameAction, length); - } - - /** - * @param content - * @return - * @throws IOException - */ - private FrameData getFrameData(ByteBuffer content) - throws IOException - { - return new FrameData(getCrc32c(content), 4); - } - - private int getCrc32c(ByteBuffer content) - { - - final int position = content.position(); - - return ((content.get(position + 3) & 0xFF) << 24) - | ((content.get(position + 2) & 0xFF) << 16) - | ((content.get(position + 1) & 0xFF) << 8) - | (content.get(position) & 0xFF); - } -} +/* + * Created: Apr 15, 2013 + */ +package org.xerial.snappy; + +import static java.lang.Math.min; +import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; +import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; +import static org.xerial.snappy.SnappyFramed.STREAM_IDENTIFIER_FLAG; +import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; +import static org.xerial.snappy.SnappyFramed.readBytes; +import static org.xerial.snappy.SnappyFramedOutputStream.MAX_BLOCK_SIZE; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.Arrays; +import java.util.zip.Checksum; + +import org.xerial.snappy.pool.BufferPool; +import org.xerial.snappy.pool.DefaultPoolFactory; + +/** + * Implements the x-snappy-framed as an {@link InputStream} and + * {@link ReadableByteChannel}. + * + * @author Brett Okken + * @since 1.1.0 + */ +public final class SnappyFramedInputStream + extends InputStream + implements + ReadableByteChannel +{ + + private final Checksum crc32 = SnappyFramed.getCRC32C(); + private final ReadableByteChannel rbc; + private final ByteBuffer frameHeader; + private final boolean verifyChecksums; + private final BufferPool bufferPool; + + /** + * A single frame read from the underlying {@link InputStream}. + */ + private ByteBuffer input; + + /** + * The decompressed data from {@link #input}. + */ + private ByteBuffer uncompressedDirect; + + /** + * Indicates if this instance has been closed. + */ + private boolean closed; + + /** + * Indicates if we have reached the EOF on {@link #input}. + */ + private boolean eof; + + /** + * The position in {@link #input} buffer to read to. + */ + private int valid; + + /** + * The next position to read from {@link #buffer}. + */ + private int position; + + /** + * Buffer contains a copy of the uncompressed data for the block. + */ + private byte[] buffer; + + /** + * Creates a Snappy input stream to read data from the specified underlying + * input stream. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param in the underlying input stream. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedInputStream(InputStream in) + throws IOException + { + this(in, true, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * input stream. + * + * @param in the underlying input stream. Must not be {@code null}. + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedInputStream(InputStream in, BufferPool bufferPool) + throws IOException + { + this(in, true, bufferPool); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * input stream. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param in the underlying input stream. Must not be {@code null}. + * @param verifyChecksums if true, checksums in input stream will be verified + * @throws IOException + */ + public SnappyFramedInputStream(InputStream in, boolean verifyChecksums) + throws IOException + { + this(in, verifyChecksums, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * input stream. + * + * @param in the underlying input stream. Must not be {@code null}. + * @param verifyChecksums if true, checksums in input stream will be verified + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedInputStream(InputStream in, boolean verifyChecksums, + BufferPool bufferPool) + throws IOException + { + this(Channels.newChannel(in), verifyChecksums, bufferPool); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * channel. + * + * @param in the underlying readable channel. Must not be {@code null}. + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedInputStream(ReadableByteChannel in, BufferPool bufferPool) + throws IOException + { + this(in, true, bufferPool); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * channel. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param in the underlying readable channel. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedInputStream(ReadableByteChannel in) + throws IOException + { + this(in, true); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * channel. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param in the underlying readable channel. Must not be {@code null}. + * @param verifyChecksums if true, checksums in input stream will be verified + * @throws IOException + */ + public SnappyFramedInputStream(ReadableByteChannel in, + boolean verifyChecksums) + throws IOException + { + this(in, verifyChecksums, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a Snappy input stream to read data from the specified underlying + * channel. + * + * @param in the underlying readable channel. Must not be {@code null}. + * @param verifyChecksums if true, checksums in input stream will be verified + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedInputStream(ReadableByteChannel in, + boolean verifyChecksums, BufferPool bufferPool) + throws IOException + { + if (in == null) { + throw new NullPointerException("in is null"); + } + + if (bufferPool == null) { + throw new NullPointerException("bufferPool is null"); + } + + this.bufferPool = bufferPool; + this.rbc = in; + this.verifyChecksums = verifyChecksums; + + allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5); + this.frameHeader = ByteBuffer.allocate(4); + + // stream must begin with stream header + final byte[] expectedHeader = HEADER_BYTES; + final byte[] actualHeader = new byte[expectedHeader.length]; + final ByteBuffer actualBuffer = ByteBuffer.wrap(actualHeader); + + final int read = SnappyFramed.readBytes(in, actualBuffer); + if (read < expectedHeader.length) { + throw new EOFException( + "encountered EOF while reading stream header"); + } + if (!Arrays.equals(expectedHeader, actualHeader)) { + throw new IOException("invalid stream header"); + } + } + + /** + * @param size + */ + private void allocateBuffersBasedOnSize(int size) + { + if (input != null) { + bufferPool.releaseDirect(input); + } + + if (uncompressedDirect != null) { + bufferPool.releaseDirect(uncompressedDirect); + } + + if (buffer != null) { + bufferPool.releaseArray(buffer); + } + + input = bufferPool.allocateDirect(size); + final int maxCompressedLength = Snappy.maxCompressedLength(size); + uncompressedDirect = bufferPool.allocateDirect(maxCompressedLength); + buffer = bufferPool.allocateArray(maxCompressedLength); + } + + @Override + public int read() + throws IOException + { + if (closed) { + return -1; + } + if (!ensureBuffer()) { + return -1; + } + return buffer[position++] & 0xFF; + } + + @Override + public int read(byte[] output, int offset, int length) + throws IOException + { + + if (output == null) { + throw new IllegalArgumentException("output is null"); + } + + if (offset < 0 || length < 0 || offset + length > output.length) { + throw new IllegalArgumentException("invalid offset [" + offset + + "] and length [" + length + ']'); + } + + if (closed) { + throw new ClosedChannelException(); + } + + if (length == 0) { + return 0; + } + if (!ensureBuffer()) { + return -1; + } + + final int size = min(length, available()); + System.arraycopy(buffer, position, output, offset, size); + position += size; + return size; + } + + @Override + public int available() + throws IOException + { + if (closed) { + return 0; + } + return valid - position; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isOpen() + { + return !closed; + } + + /** + * {@inheritDoc} + */ + @Override + public int read(ByteBuffer dst) + throws IOException + { + + if (dst == null) { + throw new IllegalArgumentException("dst is null"); + } + + if (closed) { + throw new ClosedChannelException(); + } + + if (dst.remaining() == 0) { + return 0; + } + if (!ensureBuffer()) { + return -1; + } + + final int size = min(dst.remaining(), available()); + dst.put(buffer, position, size); + position += size; + return size; + } + + /** + * Transfers the entire content of this {@link InputStream} to os. + * This potentially limits the amount of buffering required to decompress + * content. + *

+ * Unlike {@link #read(byte[], int, int)}, this method does not need to be + * called multiple times. A single call will transfer all available content. + * Any calls after the source has been exhausted will result in a return + * value of {@code 0}. + *

+ * + * @param os The destination to write decompressed content to. + * @return The number of bytes transferred. + * @throws IOException + * @since 1.1.1 + */ + public long transferTo(OutputStream os) + throws IOException + { + if (os == null) { + throw new IllegalArgumentException("os is null"); + } + + if (closed) { + throw new ClosedChannelException(); + } + + long totTransfered = 0; + + while (ensureBuffer()) { + final int available = available(); + os.write(buffer, position, available); + position += available; + totTransfered += available; + } + + return totTransfered; + } + + /** + * Transfers the entire content of this {@link ReadableByteChannel} to + * wbc. This potentially limits the amount of buffering required to + * decompress content. + *

+ *

+ * Unlike {@link #read(ByteBuffer)}, this method does not need to be called + * multiple times. A single call will transfer all available content. Any + * calls after the source has been exhausted will result in a return value + * of {@code 0}. + *

+ * + * @param wbc The destination to write decompressed content to. + * @return The number of bytes transferred. + * @throws IOException + * @since 1.1.1 + */ + public long transferTo(WritableByteChannel wbc) + throws IOException + { + if (wbc == null) { + throw new IllegalArgumentException("wbc is null"); + } + + if (closed) { + throw new ClosedChannelException(); + } + + final ByteBuffer bb = ByteBuffer.wrap(buffer); + + long totTransfered = 0; + + while (ensureBuffer()) { + bb.clear(); + bb.position(position); + bb.limit(position + available()); + + wbc.write(bb); + + final int written = bb.position() - position; + position += written; + + totTransfered += written; + } + + return totTransfered; + } + + @Override + public void close() + throws IOException + { + try { + rbc.close(); + } + finally { + if (!closed) { + closed = true; + + if (input != null) { + bufferPool.releaseDirect(input); + input = null; + } + + if (uncompressedDirect != null) { + bufferPool.releaseDirect(uncompressedDirect); + uncompressedDirect = null; + } + + if (buffer != null) { + bufferPool.releaseArray(buffer); + buffer = null; + } + } + } + } + + static enum FrameAction + { + RAW, SKIP, UNCOMPRESS; + } + + public static final class FrameMetaData + { + final int length; + final FrameAction frameAction; + + /** + * @param frameAction + * @param length + */ + public FrameMetaData(FrameAction frameAction, int length) + { + super(); + this.frameAction = frameAction; + this.length = length; + } + } + + public static final class FrameData + { + final int checkSum; + final int offset; + + /** + * @param checkSum + * @param offset + */ + public FrameData(int checkSum, int offset) + { + super(); + this.checkSum = checkSum; + this.offset = offset; + } + } + + private boolean ensureBuffer() + throws IOException + { + if (available() > 0) { + return true; + } + if (eof) { + return false; + } + + if (!readBlockHeader()) { + eof = true; + return false; + } + + // get action based on header + final FrameMetaData frameMetaData = getFrameMetaData(frameHeader); + + if (FrameAction.SKIP == frameMetaData.frameAction) { + SnappyFramed.skip(rbc, frameMetaData.length, + ByteBuffer.wrap(buffer)); + return ensureBuffer(); + } + + if (frameMetaData.length > input.capacity()) { + allocateBuffersBasedOnSize(frameMetaData.length); + } + + input.clear(); + input.limit(frameMetaData.length); + + final int actualRead = readBytes(rbc, input); + if (actualRead != frameMetaData.length) { + throw new EOFException("unexpectd EOF when reading frame"); + } + input.flip(); + + final FrameData frameData = getFrameData(input); + + if (FrameAction.UNCOMPRESS == frameMetaData.frameAction) { + + input.position(frameData.offset); + + final int uncompressedLength = Snappy.uncompressedLength(input); + + if (uncompressedLength > uncompressedDirect.capacity()) { + bufferPool.releaseDirect(uncompressedDirect); + bufferPool.releaseArray(buffer); + uncompressedDirect = bufferPool.allocateDirect(uncompressedLength); + buffer = bufferPool.allocateArray(uncompressedLength); + } + + uncompressedDirect.clear(); + + this.valid = Snappy.uncompress(input, uncompressedDirect); + + uncompressedDirect.get(buffer, 0, valid); + this.position = 0; + } + else { + // we need to start reading at the offset + input.position(frameData.offset); + this.position = 0; + this.valid = input.remaining(); + this.input.get(buffer, 0, input.remaining()); + } + + if (verifyChecksums) { + final int actualCrc32c = SnappyFramed.maskedCrc32c(crc32, buffer, + position, valid - position); + if (frameData.checkSum != actualCrc32c) { + throw new IOException("Corrupt input: invalid checksum"); + } + } + + return true; + } + + private boolean readBlockHeader() + throws IOException + { + frameHeader.clear(); + int read = readBytes(rbc, frameHeader); + + if (read == -1) { + return false; + } + + if (read < frameHeader.capacity()) { + throw new EOFException("encountered EOF while reading block header"); + } + frameHeader.flip(); + + return true; + } + + /** + * @param frameHeader + * @return + * @throws IOException + */ + private FrameMetaData getFrameMetaData(ByteBuffer frameHeader) + throws IOException + { + + assert frameHeader.hasArray(); + + final byte[] frameHeaderArray = frameHeader.array(); + + int length = (frameHeaderArray[1] & 0xFF); + length |= (frameHeaderArray[2] & 0xFF) << 8; + length |= (frameHeaderArray[3] & 0xFF) << 16; + + int minLength = 0; + final FrameAction frameAction; + final int flag = frameHeaderArray[0] & 0xFF; + switch (flag) { + case COMPRESSED_DATA_FLAG: + frameAction = FrameAction.UNCOMPRESS; + minLength = 5; + break; + case UNCOMPRESSED_DATA_FLAG: + frameAction = FrameAction.RAW; + minLength = 5; + break; + case STREAM_IDENTIFIER_FLAG: + if (length != 6) { + throw new IOException( + "stream identifier chunk with invalid length: " + + length); + } + frameAction = FrameAction.SKIP; + minLength = 6; + break; + default: + // Reserved unskippable chunks (chunk types 0x02-0x7f) + if (flag <= 0x7f) { + throw new IOException("unsupported unskippable chunk: " + + Integer.toHexString(flag)); + } + + // all that is left is Reserved skippable chunks (chunk types + // 0x80-0xfe) + frameAction = FrameAction.SKIP; + minLength = 0; + } + + if (length < minLength) { + throw new IOException("invalid length: " + length + + " for chunk flag: " + Integer.toHexString(flag)); + } + + return new FrameMetaData(frameAction, length); + } + + /** + * @param content + * @return + * @throws IOException + */ + private FrameData getFrameData(ByteBuffer content) + throws IOException + { + // the first 4 bytes are the crc32c value in little endian order + content.order(ByteOrder.LITTLE_ENDIAN); + final int crc32c = content.getInt(content.position()); + return new FrameData(crc32c, 4); + } +} diff --git a/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java b/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java index 7d4d1a7e..409a11e1 100644 --- a/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java @@ -1,474 +1,570 @@ -/* - * Created: Apr 12, 2013 - */ -package org.xerial.snappy; - -import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; -import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; -import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; -import static org.xerial.snappy.SnappyFramed.maskedCrc32c; -import static org.xerial.snappy.SnappyFramed.releaseDirectByteBuffer; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; - -/** - * Implements the x-snappy-framed as an {@link OutputStream} and - * {@link WritableByteChannel}. - * - * @author Brett Okken - * @since 1.1.0 - */ -public final class SnappyFramedOutputStream - extends OutputStream - implements - WritableByteChannel -{ - - /** - * The x-snappy-framed specification allows for a chunk size up to - * 16,777,211 bytes in length. However, it also goes on to state: - *

- * - * We place an additional restriction that the uncompressed data in a chunk - * must be no longer than 65536 bytes. This allows consumers to easily use - * small fixed-size buffers. - * - *

- */ - public static final int MAX_BLOCK_SIZE = 64 * 1024; - - /** - * The default block size to use. - */ - public static final int DEFAULT_BLOCK_SIZE = MAX_BLOCK_SIZE; - - /** - * The default min compression ratio to use. - */ - public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d; - - private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order( - ByteOrder.LITTLE_ENDIAN); - private final ByteBuffer buffer; - private final ByteBuffer directInputBuffer; - private final ByteBuffer outputBuffer; - private final double minCompressionRatio; - - private final WritableByteChannel out; - - // private int position; - private boolean closed; - - /** - * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE} - * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. - * - * @param out The underlying {@link OutputStream} to write to. Must not be - * {@code null}. - * @throws IOException - */ - public SnappyFramedOutputStream(OutputStream out) - throws IOException - { - this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO); - } - - /** - * Creates a new {@link SnappyFramedOutputStream} instance. - * - * @param out The underlying {@link OutputStream} to write to. Must not be - * {@code null}. - * @param blockSize The block size (of raw data) to compress before writing frames - * to out. Must be in (0, 65536]. - * @param minCompressionRatio Defines the minimum compression ratio ( - * {@code compressedLength / rawLength}) that must be achieved to - * write the compressed data. This must be in (0, 1.0]. - * @throws IOException - */ - public SnappyFramedOutputStream(OutputStream out, int blockSize, - double minCompressionRatio) - throws IOException - { - this(Channels.newChannel(out), blockSize, minCompressionRatio); - } - - /** - * Creates a new {@link SnappyFramedOutputStream} using the - * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. - * - * @param out The underlying {@link WritableByteChannel} to write to. Must - * not be {@code null}. - * @throws IOException - * @since 1.1.1 - */ - public SnappyFramedOutputStream(WritableByteChannel out) - throws IOException - { - this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO); - } - - /** - * Creates a new {@link SnappyFramedOutputStream} instance. - * - * @param out The underlying {@link WritableByteChannel} to write to. Must - * not be {@code null}. - * @param blockSize The block size (of raw data) to compress before writing frames - * to out. Must be in (0, 65536]. - * @param minCompressionRatio Defines the minimum compression ratio ( - * {@code compressedLength / rawLength}) that must be achieved to - * write the compressed data. This must be in (0, 1.0]. - * @throws IOException - * @since 1.1.1 - */ - public SnappyFramedOutputStream(WritableByteChannel out, int blockSize, - double minCompressionRatio) - throws IOException - { - if (out == null) { - throw new NullPointerException(); - } - - if (minCompressionRatio <= 0 || minCompressionRatio > 1.0) { - throw new IllegalArgumentException("minCompressionRatio " - + minCompressionRatio + " must be in (0,1.0]"); - } - - if (blockSize <= 0 || blockSize > MAX_BLOCK_SIZE) { - throw new IllegalArgumentException("block size " + blockSize - + " must be in (0, 65536]"); - } - - this.out = out; - this.minCompressionRatio = minCompressionRatio; - buffer = ByteBuffer.allocate(blockSize); - directInputBuffer = ByteBuffer.allocateDirect(blockSize); - outputBuffer = ByteBuffer.allocateDirect(Snappy - .maxCompressedLength(blockSize)); - - writeHeader(out); - } - - /** - * Writes the implementation specific header or "marker bytes" to - * out. - * - * @param out The underlying {@link OutputStream}. - * @throws IOException - */ - private void writeHeader(WritableByteChannel out) - throws IOException - { - out.write(ByteBuffer.wrap(HEADER_BYTES)); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isOpen() - { - return !closed; - } - - @Override - public void write(int b) - throws IOException - { - if (closed) { - throw new IOException("Stream is closed"); - } - if (buffer.remaining() <= 0) { - flushBuffer(); - } - buffer.put((byte) b); - } - - @Override - public void write(byte[] input, int offset, int length) - throws IOException - { - if (closed) { - throw new IOException("Stream is closed"); - } - - if (input == null) { - throw new NullPointerException(); - } - else if ((offset < 0) || (offset > input.length) || (length < 0) - || ((offset + length) > input.length) - || ((offset + length) < 0)) { - throw new IndexOutOfBoundsException(); - } - - while (length > 0) { - if (buffer.remaining() <= 0) { - flushBuffer(); - } - - final int toPut = Math.min(length, buffer.remaining()); - buffer.put(input, offset, toPut); - offset += toPut; - length -= toPut; - } - } - - /** - * {@inheritDoc} - */ - @Override - public int write(ByteBuffer src) - throws IOException - { - if (closed) { - throw new ClosedChannelException(); - } - - if (buffer.remaining() <= 0) { - flushBuffer(); - } - - final int srcLength = src.remaining(); - - // easy case: enough free space in buffer for entire input - if (buffer.remaining() >= src.remaining()) { - buffer.put(src); - return srcLength; - } - - // store current limit - final int srcEnd = src.position() + src.remaining(); - - while ((src.position() + buffer.remaining()) <= srcEnd) { - // fill partial buffer as much as possible and flush - src.limit(src.position() + buffer.remaining()); - buffer.put(src); - flushBuffer(); - } - - // reset original limit - src.limit(srcEnd); - - // copy remaining partial block into now-empty buffer - buffer.put(src); - - return srcLength; - } - - /** - * Transfers all the content from is to this {@link OutputStream}. - * This potentially limits the amount of buffering required to compress - * content. - * - * @param is The source of data to compress. - * @return The number of bytes read from is. - * @throws IOException - * @since 1.1.1 - */ - public long transferFrom(InputStream is) - throws IOException - { - if (closed) { - throw new ClosedChannelException(); - } - - if (is == null) { - throw new NullPointerException(); - } - - if (buffer.remaining() == 0) { - flushBuffer(); - } - - assert buffer.hasArray(); - final byte[] bytes = buffer.array(); - - final int arrayOffset = buffer.arrayOffset(); - long totTransfered = 0; - int read; - while ((read = is.read(bytes, arrayOffset + buffer.position(), - buffer.remaining())) != -1) { - buffer.position(buffer.position() + read); - - if (buffer.remaining() == 0) { - flushBuffer(); - } - - totTransfered += read; - } - - return totTransfered; - } - - /** - * Transfers all the content from rbc to this - * {@link WritableByteChannel}. This potentially limits the amount of - * buffering required to compress content. - * - * @param rbc The source of data to compress. - * @return The number of bytes read from rbc. - * @throws IOException - * @since 1.1.1 - */ - public long transferFrom(ReadableByteChannel rbc) - throws IOException - { - if (closed) { - throw new ClosedChannelException(); - } - - if (rbc == null) { - throw new NullPointerException(); - } - - if (buffer.remaining() == 0) { - flushBuffer(); - } - - long totTransfered = 0; - int read; - while ((read = rbc.read(buffer)) != -1) { - if (buffer.remaining() == 0) { - flushBuffer(); - } - - totTransfered += read; - } - - return totTransfered; - } - - @Override - public final void flush() - throws IOException - { - if (closed) { - throw new IOException("Stream is closed"); - } - flushBuffer(); - } - - @Override - public final void close() - throws IOException - { - if (closed) { - return; - } - try { - flush(); - out.close(); - } - finally { - closed = true; - - releaseDirectByteBuffer(directInputBuffer); - releaseDirectByteBuffer(outputBuffer); - } - } - - /** - * Compresses and writes out any buffered data. This does nothing if there - * is no currently buffered data. - * - * @throws IOException - */ - private void flushBuffer() - throws IOException - { - if (buffer.position() > 0) { - buffer.flip(); - writeCompressed(buffer); - buffer.clear(); - } - } - - /** - * {@link SnappyFramed#maskedCrc32c(byte[], int, int)} the crc, compresses - * the data, determines if the compression ratio is acceptable and calls - * {@link #writeBlock(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer, boolean, int)} to - * actually write the frame. - * - * @param buffer - * @throws IOException - */ - private void writeCompressed(ByteBuffer buffer) - throws IOException - { - - final byte[] input = buffer.array(); - final int length = buffer.remaining(); - - // crc is based on the user supplied input data - final int crc32c = maskedCrc32c(input, 0, length); - - directInputBuffer.clear(); - directInputBuffer.put(buffer); - directInputBuffer.flip(); - - outputBuffer.clear(); - Snappy.compress(directInputBuffer, outputBuffer); - - final int compressedLength = outputBuffer.remaining(); - - // only use the compressed data if compression ratio is <= the - // minCompressonRatio - if (((double) compressedLength / (double) length) <= minCompressionRatio) { - writeBlock(out, outputBuffer, true, crc32c); - } - else { - // otherwise use the uncompressed data. - buffer.flip(); - writeBlock(out, buffer, false, crc32c); - } - } - - /** - * Write a frame (block) to out. - * - * @param out The {@link OutputStream} to write to. - * @param data The data to write. - * @param compressed Indicates if data is the compressed or raw content. - * This is based on whether the compression ratio desired is - * reached. - * @param crc32c The calculated checksum. - * @throws IOException - */ - private void writeBlock(final WritableByteChannel out, ByteBuffer data, - boolean compressed, int crc32c) - throws IOException - { - - headerBuffer.clear(); - headerBuffer.put((byte) (compressed ? COMPRESSED_DATA_FLAG - : UNCOMPRESSED_DATA_FLAG)); - - // the length written out to the header is both the checksum and the - // frame - final int headerLength = data.remaining() + 4; - - // write length - headerBuffer.put((byte) headerLength); - headerBuffer.put((byte) (headerLength >>> 8)); - headerBuffer.put((byte) (headerLength >>> 16)); - - // write crc32c of user input data - headerBuffer.putInt(crc32c); - - headerBuffer.flip(); - - // write the header - out.write(headerBuffer); - // write the raw data - out.write(data); - } -} +/* + * Created: Apr 12, 2013 + */ +package org.xerial.snappy; + +import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; +import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; +import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; +import static org.xerial.snappy.SnappyFramed.maskedCrc32c; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.zip.Checksum; + +import org.xerial.snappy.pool.BufferPool; +import org.xerial.snappy.pool.DefaultPoolFactory; + +/** + * Implements the x-snappy-framed as an {@link OutputStream} and + * {@link WritableByteChannel}. + * + * @author Brett Okken + * @since 1.1.0 + */ +public final class SnappyFramedOutputStream + extends OutputStream + implements + WritableByteChannel +{ + + /** + * The x-snappy-framed specification allows for a chunk size up to + * 16,777,211 bytes in length. However, it also goes on to state: + *

+ * + * We place an additional restriction that the uncompressed data in a chunk + * must be no longer than 65536 bytes. This allows consumers to easily use + * small fixed-size buffers. + * + *

+ */ + public static final int MAX_BLOCK_SIZE = 64 * 1024; + + /** + * The default block size to use. + */ + public static final int DEFAULT_BLOCK_SIZE = MAX_BLOCK_SIZE; + + /** + * The default min compression ratio to use. + */ + public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d; + + private final Checksum crc32 = SnappyFramed.getCRC32C(); + private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order( + ByteOrder.LITTLE_ENDIAN); + private final BufferPool bufferPool; + private final int blockSize; + private final ByteBuffer buffer; + private final ByteBuffer directInputBuffer; + private final ByteBuffer outputBuffer; + private final double minCompressionRatio; + + private final WritableByteChannel out; + + // private int position; + private boolean closed; + + /** + * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE} + * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param out The underlying {@link OutputStream} to write to. Must not be + * {@code null}. + * @throws IOException + */ + public SnappyFramedOutputStream(OutputStream out) + throws IOException + { + this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE} + * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. + * + * @param out The underlying {@link OutputStream} to write to. Must not be + * {@code null}. + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedOutputStream(OutputStream out, BufferPool bufferPool) + throws IOException + { + this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, bufferPool); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} instance. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param out The underlying {@link OutputStream} to write to. Must not be + * {@code null}. + * @param blockSize The block size (of raw data) to compress before writing frames + * to out. Must be in (0, 65536]. + * @param minCompressionRatio Defines the minimum compression ratio ( + * {@code compressedLength / rawLength}) that must be achieved to + * write the compressed data. This must be in (0, 1.0]. + * @throws IOException + */ + public SnappyFramedOutputStream(OutputStream out, int blockSize, + double minCompressionRatio) + throws IOException + { + this(Channels.newChannel(out), blockSize, minCompressionRatio, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} instance. + * + * @param out The underlying {@link OutputStream} to write to. Must not be + * {@code null}. + * @param blockSize The block size (of raw data) to compress before writing frames + * to out. Must be in (0, 65536]. + * @param minCompressionRatio Defines the minimum compression ratio ( + * {@code compressedLength / rawLength}) that must be achieved to + * write the compressed data. This must be in (0, 1.0]. + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedOutputStream(OutputStream out, int blockSize, + double minCompressionRatio, BufferPool bufferPool) + throws IOException + { + this(Channels.newChannel(out), blockSize, minCompressionRatio, bufferPool); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} using the + * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param out The underlying {@link WritableByteChannel} to write to. Must + * not be {@code null}. + * @throws IOException + * @since 1.1.1 + */ + public SnappyFramedOutputStream(WritableByteChannel out) + throws IOException + { + this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} using the + * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. + *

+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers. + *

+ * + * @param out The underlying {@link WritableByteChannel} to write to. Must + * not be {@code null}. + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + */ + public SnappyFramedOutputStream(WritableByteChannel out, BufferPool bufferPool) + throws IOException + { + this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, bufferPool); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} instance. + * + * @param out The underlying {@link WritableByteChannel} to write to. Must + * not be {@code null}. + * @param blockSize The block size (of raw data) to compress before writing frames + * to out. Must be in (0, 65536]. + * @param minCompressionRatio Defines the minimum compression ratio ( + * {@code compressedLength / rawLength}) that must be achieved to + * write the compressed data. This must be in (0, 1.0]. + * @throws IOException + * @since 1.1.1 + */ + public SnappyFramedOutputStream(WritableByteChannel out, int blockSize, + double minCompressionRatio) + throws IOException + { + this(out, blockSize, minCompressionRatio, DefaultPoolFactory.getDefaultPool()); + } + + /** + * Creates a new {@link SnappyFramedOutputStream} instance. + * + * @param out The underlying {@link WritableByteChannel} to write to. Must + * not be {@code null}. + * @param blockSize The block size (of raw data) to compress before writing frames + * to out. Must be in (0, 65536]. + * @param minCompressionRatio Defines the minimum compression ratio ( + * {@code compressedLength / rawLength}) that must be achieved to + * write the compressed data. This must be in (0, 1.0]. + * @param bufferPool Used to obtain buffer instances. Must not be {@code null}. + * @throws IOException + * @since 1.1.1 + */ + public SnappyFramedOutputStream(WritableByteChannel out, int blockSize, + double minCompressionRatio, BufferPool bufferPool) + throws IOException + { + if (out == null) { + throw new NullPointerException("out is null"); + } + + if (bufferPool == null) { + throw new NullPointerException("buffer pool is null"); + } + + if (minCompressionRatio <= 0 || minCompressionRatio > 1.0) { + throw new IllegalArgumentException("minCompressionRatio " + + minCompressionRatio + " must be in (0,1.0]"); + } + + if (blockSize <= 0 || blockSize > MAX_BLOCK_SIZE) { + throw new IllegalArgumentException("block size " + blockSize + + " must be in (0, 65536]"); + } + this.blockSize = blockSize; + this.out = out; + this.minCompressionRatio = minCompressionRatio; + + this.bufferPool = bufferPool; + buffer = ByteBuffer.wrap(bufferPool.allocateArray(blockSize), 0, blockSize); + directInputBuffer = bufferPool.allocateDirect(blockSize); + outputBuffer = bufferPool.allocateDirect(Snappy + .maxCompressedLength(blockSize)); + + writeHeader(out); + } + + /** + * Writes the implementation specific header or "marker bytes" to + * out. + * + * @param out The underlying {@link OutputStream}. + * @throws IOException + */ + private void writeHeader(WritableByteChannel out) + throws IOException + { + out.write(ByteBuffer.wrap(HEADER_BYTES)); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isOpen() + { + return !closed; + } + + @Override + public void write(int b) + throws IOException + { + if (closed) { + throw new IOException("Stream is closed"); + } + if (buffer.remaining() <= 0) { + flushBuffer(); + } + buffer.put((byte) b); + } + + @Override + public void write(byte[] input, int offset, int length) + throws IOException + { + if (closed) { + throw new IOException("Stream is closed"); + } + + if (input == null) { + throw new NullPointerException(); + } + else if ((offset < 0) || (offset > input.length) || (length < 0) + || ((offset + length) > input.length) + || ((offset + length) < 0)) { + throw new IndexOutOfBoundsException(); + } + + while (length > 0) { + if (buffer.remaining() <= 0) { + flushBuffer(); + } + + final int toPut = Math.min(length, buffer.remaining()); + buffer.put(input, offset, toPut); + offset += toPut; + length -= toPut; + } + } + + /** + * {@inheritDoc} + */ + @Override + public int write(ByteBuffer src) + throws IOException + { + if (closed) { + throw new ClosedChannelException(); + } + + if (buffer.remaining() <= 0) { + flushBuffer(); + } + + final int srcLength = src.remaining(); + + // easy case: enough free space in buffer for entire input + if (buffer.remaining() >= src.remaining()) { + buffer.put(src); + return srcLength; + } + + // store current limit + final int srcEnd = src.position() + src.remaining(); + + while ((src.position() + buffer.remaining()) <= srcEnd) { + // fill partial buffer as much as possible and flush + src.limit(src.position() + buffer.remaining()); + buffer.put(src); + flushBuffer(); + } + + // reset original limit + src.limit(srcEnd); + + // copy remaining partial block into now-empty buffer + buffer.put(src); + + return srcLength; + } + + /** + * Transfers all the content from is to this {@link OutputStream}. + * This potentially limits the amount of buffering required to compress + * content. + * + * @param is The source of data to compress. + * @return The number of bytes read from is. + * @throws IOException + * @since 1.1.1 + */ + public long transferFrom(InputStream is) + throws IOException + { + if (closed) { + throw new ClosedChannelException(); + } + + if (is == null) { + throw new NullPointerException(); + } + + if (buffer.remaining() == 0) { + flushBuffer(); + } + + assert buffer.hasArray(); + final byte[] bytes = buffer.array(); + + final int arrayOffset = buffer.arrayOffset(); + long totTransfered = 0; + int read; + while ((read = is.read(bytes, arrayOffset + buffer.position(), + buffer.remaining())) != -1) { + buffer.position(buffer.position() + read); + + if (buffer.remaining() == 0) { + flushBuffer(); + } + + totTransfered += read; + } + + return totTransfered; + } + + /** + * Transfers all the content from rbc to this + * {@link WritableByteChannel}. This potentially limits the amount of + * buffering required to compress content. + * + * @param rbc The source of data to compress. + * @return The number of bytes read from rbc. + * @throws IOException + * @since 1.1.1 + */ + public long transferFrom(ReadableByteChannel rbc) + throws IOException + { + if (closed) { + throw new ClosedChannelException(); + } + + if (rbc == null) { + throw new NullPointerException(); + } + + if (buffer.remaining() == 0) { + flushBuffer(); + } + + long totTransfered = 0; + int read; + while ((read = rbc.read(buffer)) != -1) { + if (buffer.remaining() == 0) { + flushBuffer(); + } + + totTransfered += read; + } + + return totTransfered; + } + + @Override + public final void flush() + throws IOException + { + if (closed) { + throw new IOException("Stream is closed"); + } + flushBuffer(); + } + + @Override + public final void close() + throws IOException + { + if (closed) { + return; + } + try { + flush(); + out.close(); + } + finally { + closed = true; + bufferPool.releaseArray(buffer.array()); + bufferPool.releaseDirect(directInputBuffer); + bufferPool.releaseDirect(outputBuffer); + } + } + + /** + * Compresses and writes out any buffered data. This does nothing if there + * is no currently buffered data. + * + * @throws IOException + */ + private void flushBuffer() + throws IOException + { + if (buffer.position() > 0) { + buffer.flip(); + writeCompressed(buffer); + buffer.clear(); + buffer.limit(blockSize); + } + } + + /** + * {@link SnappyFramed#maskedCrc32c(byte[], int, int)} the crc, compresses + * the data, determines if the compression ratio is acceptable and calls + * {@link #writeBlock(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer, boolean, int)} to + * actually write the frame. + * + * @param buffer + * @throws IOException + */ + private void writeCompressed(ByteBuffer buffer) + throws IOException + { + + final byte[] input = buffer.array(); + final int length = buffer.remaining(); + + // crc is based on the user supplied input data + final int crc32c = maskedCrc32c(crc32, input, 0, length); + + directInputBuffer.clear(); + directInputBuffer.put(buffer); + directInputBuffer.flip(); + + outputBuffer.clear(); + Snappy.compress(directInputBuffer, outputBuffer); + + final int compressedLength = outputBuffer.remaining(); + + // only use the compressed data if compression ratio is <= the + // minCompressonRatio + if (((double) compressedLength / (double) length) <= minCompressionRatio) { + writeBlock(out, outputBuffer, true, crc32c); + } + else { + // otherwise use the uncompressed data. + buffer.flip(); + writeBlock(out, buffer, false, crc32c); + } + } + + /** + * Write a frame (block) to out. + * + * @param out The {@link OutputStream} to write to. + * @param data The data to write. + * @param compressed Indicates if data is the compressed or raw content. + * This is based on whether the compression ratio desired is + * reached. + * @param crc32c The calculated checksum. + * @throws IOException + */ + private void writeBlock(final WritableByteChannel out, ByteBuffer data, + boolean compressed, int crc32c) + throws IOException + { + + headerBuffer.clear(); + headerBuffer.put((byte) (compressed ? COMPRESSED_DATA_FLAG + : UNCOMPRESSED_DATA_FLAG)); + + // the length written out to the header is both the checksum and the + // frame + final int headerLength = data.remaining() + 4; + + // write length + headerBuffer.put((byte) headerLength); + headerBuffer.put((byte) (headerLength >>> 8)); + headerBuffer.put((byte) (headerLength >>> 16)); + + // write crc32c of user input data + headerBuffer.putInt(crc32c); + + headerBuffer.flip(); + + // write the header + out.write(headerBuffer); + // write the raw data + out.write(data); + } +} diff --git a/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java b/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java index 67d2d188..d9f69bf4 100644 --- a/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java @@ -1,9 +1,9 @@ package org.xerial.snappy; -import java.io.OutputStream; - import org.xerial.snappy.buffer.CachedBufferAllocator; +import java.io.OutputStream; + public class SnappyHadoopCompatibleOutputStream extends SnappyOutputStream { public SnappyHadoopCompatibleOutputStream(OutputStream out) diff --git a/src/main/java/org/xerial/snappy/SnappyInputStream.java b/src/main/java/org/xerial/snappy/SnappyInputStream.java index 24f59767..d6ffcd43 100755 --- a/src/main/java/org/xerial/snappy/SnappyInputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyInputStream.java @@ -36,8 +36,11 @@ public class SnappyInputStream extends InputStream { + public static final int MAX_CHUNK_SIZE = 512 * 1024 * 1024; // 512 MiB + private boolean finishedReading = false; protected final InputStream in; + private final int maxChunkSize; private byte[] compressed; private byte[] uncompressed; @@ -55,6 +58,21 @@ public class SnappyInputStream public SnappyInputStream(InputStream input) throws IOException { + this(input, MAX_CHUNK_SIZE); + } + + + /** + * Create a filter for reading compressed data as a uncompressed stream with provided maximum chunk size + * + * @param input + * @param maxChunkSize + * @throws IOException + */ + public SnappyInputStream(InputStream input, int maxChunkSize) + throws IOException + { + this.maxChunkSize = maxChunkSize; this.in = input; readHeader(); } @@ -417,8 +435,19 @@ protected boolean hasNextChunk() } } + // chunkSize is negative + if (chunkSize < 0) { + throw new SnappyError(SnappyErrorCode.INVALID_CHUNK_SIZE, "chunkSize is too big or negative : " + chunkSize); + } + + // chunkSize is big + if (chunkSize > maxChunkSize) { + throw new SnappyError(SnappyErrorCode.FAILED_TO_UNCOMPRESS, String.format("Received chunkSize %,d is greater than max configured chunk size %,d", chunkSize, maxChunkSize)); + } + // extend the compressed data buffer size if (compressed == null || chunkSize > compressed.length) { + // chunkSize exceeds limit compressed = new byte[chunkSize]; } readBytes = 0; diff --git a/src/main/java/org/xerial/snappy/SnappyLoader.java b/src/main/java/org/xerial/snappy/SnappyLoader.java old mode 100755 new mode 100644 index 27855afb..f1d555cc --- a/src/main/java/org/xerial/snappy/SnappyLoader.java +++ b/src/main/java/org/xerial/snappy/SnappyLoader.java @@ -19,13 +19,15 @@ // SnappyLoader.java // Since: 2011/03/29 // -// $URL$ +// $URL$ // $Author$ //-------------------------------------- package org.xerial.snappy; import java.io.*; +import java.net.JarURLConnection; import java.net.URL; +import java.net.URLConnection; import java.util.Enumeration; import java.util.Properties; import java.util.UUID; @@ -49,13 +51,13 @@ *
  • (System property: org.xerial.snappy.lib.path)/(System property: * org.xerial.lib.name) *
  • One of the libraries embedded in snappy-java-(version).jar extracted into - * (System property: java.io.tempdir). If + * (System property: java.io.tmpdir). If * org.xerial.snappy.tempdir is set, use this folder instead of - * java.io.tempdir. + * java.io.tmpdir. * *

    *

    - * If you do not want to use folder java.io.tempdir, set the System + * If you do not want to use folder java.io.tmpdir, set the System * property org.xerial.snappy.tempdir. For example, to use * /tmp/leo as a temporary folder to copy native libraries, use -D option * of JVM: @@ -75,13 +77,14 @@ public class SnappyLoader public static final String SNAPPY_SYSTEM_PROPERTIES_FILE = "org-xerial-snappy.properties"; public static final String KEY_SNAPPY_LIB_PATH = "org.xerial.snappy.lib.path"; public static final String KEY_SNAPPY_LIB_NAME = "org.xerial.snappy.lib.name"; + public static final String KEY_SNAPPY_PUREJAVA = "org.xerial.snappy.purejava"; public static final String KEY_SNAPPY_TEMPDIR = "org.xerial.snappy.tempdir"; public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib"; public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility private static boolean isLoaded = false; - private static volatile SnappyNative snappyApi = null; + private static volatile SnappyApi snappyApi = null; private static volatile BitShuffleNative bitshuffleApi = null; private static File nativeLibFile = null; @@ -101,11 +104,11 @@ static void cleanUpExtractedNativeLib() /** * Set the `snappyApi` instance. * - * @param nativeCode + * @param apiImpl */ - static synchronized void setSnappyApi(SnappyNative nativeCode) + static synchronized void setSnappyApi(SnappyApi apiImpl) { - snappyApi = nativeCode; + snappyApi = apiImpl; } /** @@ -146,7 +149,7 @@ private static void loadSnappySystemProperties() loadSnappySystemProperties(); } - static synchronized SnappyNative loadSnappyApi() + static synchronized SnappyApi loadSnappyApi() { if (snappyApi != null) { return snappyApi; @@ -234,7 +237,7 @@ private static File extractLibraryFile(String libFolderForCurrentOS, String libr InputStream reader = null; FileOutputStream writer = null; try { - reader = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath); + reader = getResourceAsInputStream(nativeLibraryFilePath); try { writer = new FileOutputStream(extractedLibFile); @@ -272,7 +275,7 @@ private static File extractLibraryFile(String libFolderForCurrentOS, String libr InputStream nativeIn = null; InputStream extractedLibIn = null; try { - nativeIn = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath); + nativeIn = getResourceAsInputStream(nativeLibraryFilePath); extractedLibIn = new FileInputStream(extractedLibFile); if (!contentsEquals(nativeIn, extractedLibIn)) { @@ -311,7 +314,7 @@ static File findNativeLibrary() String snappyNativeLibraryPath = System.getProperty(KEY_SNAPPY_LIB_PATH); String snappyNativeLibraryName = System.getProperty(KEY_SNAPPY_LIB_NAME); - // Resolve the library file name with a suffix (e.g., dll, .so, etc.) + // Resolve the library file name with a suffix (e.g., dll, .so, etc.) if (snappyNativeLibraryName == null) { snappyNativeLibraryName = System.mapLibraryName("snappyjava"); } @@ -329,7 +332,7 @@ static File findNativeLibrary() if (!hasNativeLib) { if (OSInfo.getOSName().equals("Mac")) { // Fix for openjdk7 for Mac - String altName = "libsnappyjava.jnilib"; + String altName = "libsnappyjava.dylib"; if (hasResource(snappyNativeLibraryPath + "/" + altName)) { snappyNativeLibraryName = altName; hasNativeLib = true; @@ -393,4 +396,16 @@ public static String getVersion() } return version; } + + private static InputStream getResourceAsInputStream(String resourcePath) throws IOException { + URL url = SnappyLoader.class.getResource(resourcePath); + URLConnection connection = url.openConnection(); + if (connection instanceof JarURLConnection) { + JarURLConnection jarConnection = (JarURLConnection) connection; + jarConnection.setUseCaches(false); // workaround for https://bugs.openjdk.org/browse/JDK-8205976 + return jarConnection.getInputStream(); + } else { + return connection.getInputStream(); + } + } } diff --git a/src/main/java/org/xerial/snappy/SnappyNative.cpp b/src/main/java/org/xerial/snappy/SnappyNative.cpp index aacd6290..496ba13b 100755 --- a/src/main/java/org/xerial/snappy/SnappyNative.cpp +++ b/src/main/java/org/xerial/snappy/SnappyNative.cpp @@ -20,7 +20,7 @@ inline void throw_exception(JNIEnv *env, jobject self, int errorCode) { - jclass c = env->FindClass("org/xerial/snappy/SnappyNative"); + jclass c = env->GetObjectClass(self); if(c==0) return; jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V"); diff --git a/src/main/java/org/xerial/snappy/SnappyNative.java b/src/main/java/org/xerial/snappy/SnappyNative.java index 95a6f419..98ae1d9a 100755 --- a/src/main/java/org/xerial/snappy/SnappyNative.java +++ b/src/main/java/org/xerial/snappy/SnappyNative.java @@ -19,7 +19,7 @@ // SnappyNative.java // Since: 2011/03/30 // -// $URL$ +// $URL$ // $Author$ //-------------------------------------- package org.xerial.snappy; @@ -38,7 +38,7 @@ * * @author leo */ -public class SnappyNative +public class SnappyNative implements SnappyApi { public native String nativeLibraryVersion(); @@ -46,49 +46,63 @@ public class SnappyNative // ------------------------------------------------------------------------ // Generic compression/decompression routines. // ------------------------------------------------------------------------ + @Override public native long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException; + @Override public native long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException; + @Override public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, int outputOffset) throws IOException; + @Override public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) throws IOException; + @Override public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed, int outputOffset) throws IOException; + @Override public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset) throws IOException; // Returns the maximal size of the compressed representation of // input data that is "source_bytes" bytes in length; + @Override public native int maxCompressedLength(int source_bytes); // This operation takes O(1) time. + @Override public native int uncompressedLength(ByteBuffer compressed, int offset, int len) throws IOException; + @Override public native int uncompressedLength(Object input, int offset, int len) throws IOException; + @Override public native long uncompressedLength(long inputAddr, long len) throws IOException; + @Override public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) throws IOException; + @Override public native boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException; + @Override public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len) throws IOException; + @Override public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException; diff --git a/src/main/java/org/xerial/snappy/SnappyOutputStream.java b/src/main/java/org/xerial/snappy/SnappyOutputStream.java index da47bf1b..c3a5f386 100755 --- a/src/main/java/org/xerial/snappy/SnappyOutputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyOutputStream.java @@ -59,8 +59,9 @@ public class SnappyOutputStream extends OutputStream { - static final int MIN_BLOCK_SIZE = 1 * 1024; - static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size + public static final int MAX_BLOCK_SIZE = 512 * 1024 * 1024; // 512 MiB + public static final int MIN_BLOCK_SIZE = 1 * 1024; + public static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size protected final OutputStream out; private final int blockSize; @@ -84,7 +85,7 @@ public SnappyOutputStream(OutputStream out) /** * @param out * @param blockSize byte size of the internal buffer size - * @throws IOException + * @throws IllegalArgumentException when blockSize is larger than 512 MiB */ public SnappyOutputStream(OutputStream out, int blockSize) { @@ -95,6 +96,9 @@ public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactor { this.out = out; this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize); + if (this.blockSize > MAX_BLOCK_SIZE){ + throw new IllegalArgumentException(String.format("Provided chunk size %,d larger than max %,d", this.blockSize, MAX_BLOCK_SIZE)); + } int inputSize = blockSize; int outputSize = SnappyCodec.HEADER_SIZE + 4 + Snappy.maxCompressedLength(blockSize); @@ -365,22 +369,18 @@ protected void dumpOutput() protected void compressInput() throws IOException { - if (inputCursor <= 0) { - return; // no need to dump - } - if (!headerWritten) { outputCursor = writeHeader(); headerWritten = true; } - + if (inputCursor <= 0) { + return; // no need to dump + } // Compress and dump the buffer content if (!hasSufficientOutputBufferFor(inputCursor)) { dumpOutput(); } - writeBlockPreemble(); - int compressedSize = Snappy.compress(inputBuffer, 0, inputCursor, outputBuffer, outputCursor + 4); // Write compressed data size writeInt(outputBuffer, outputCursor, compressedSize); diff --git a/src/main/java/org/xerial/snappy/pool/BufferPool.java b/src/main/java/org/xerial/snappy/pool/BufferPool.java new file mode 100644 index 00000000..ae39a767 --- /dev/null +++ b/src/main/java/org/xerial/snappy/pool/BufferPool.java @@ -0,0 +1,53 @@ +package org.xerial.snappy.pool; + +import java.nio.ByteBuffer; + +/** + * Makes various types of buffers available for use and potential re-use. + * + *

    + * Implementations must be safe for concurrent use by multiple threads. + *

    + * + * @author Brett Okken + */ +public interface BufferPool { + + /** + * Returns a {@code byte[]} of size or greater length. + * @param size The minimum size array required. Must be {@code >= 0}. + * @return A {@code byte[]} with length of at least size. + * @see #releaseArray(byte[]) + */ + public byte[] allocateArray(int size); + + /** + * Returns instance to pool for potential future reuse. + *

    + * Must not be returned more than 1 time. Must not be used by caller after return. + *

    + * @param buffer Instance to return to pool. Must not be {@code null}. + * Must not be returned more than 1 time. Must not be used by caller after return. + */ + public void releaseArray(byte[] buffer); + + /** + * Returns a {@link ByteBuffer#allocateDirect(int) direct ByteBuffer} of size or + * greater {@link ByteBuffer#capacity() capacity}. + * @param size The minimum size buffer required. Must be {@code >= 0}. + * @return A {@code ByteBuffer} of size or greater {@link ByteBuffer#capacity() capacity}. + * @see #releaseDirect(ByteBuffer) + * @see ByteBuffer#allocateDirect(int) + */ + public ByteBuffer allocateDirect(int size); + + /** + * Returns instance to pool for potential future reuse. + *

    + * Must not be returned more than 1 time. Must not be used by caller after return. + *

    + * @param buffer Instance to return to pool. Must not be {@code null}. + * Must not be returned more than 1 time. Must not be used by caller after return. + */ + public void releaseDirect(ByteBuffer buffer); +} diff --git a/src/main/java/org/xerial/snappy/pool/CachingBufferPool.java b/src/main/java/org/xerial/snappy/pool/CachingBufferPool.java new file mode 100644 index 00000000..80232e33 --- /dev/null +++ b/src/main/java/org/xerial/snappy/pool/CachingBufferPool.java @@ -0,0 +1,216 @@ +package org.xerial.snappy.pool; + +import java.lang.ref.SoftReference; +import java.nio.ByteBuffer; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentMap; + +/** + * A {@link BufferPool} implementation which caches values at fixed sizes. + *

    + * Pooled instances are held as {@link SoftReference} to allow GC if necessary. + *

    + *

    + * The current fixed sizes are calculated as follows: + *

      + *
    • Values < 4KB return 4KB
      • + *
    • 4KB - 32KB to 2KB
      • + *
    • 32KB - 512KB to 16KB
      • + *
    • 512KB - 2MB to 128KB
      • + *
    • 2MB - 16MB to 512KB
      • + *
    • 16MB - 128MB to 4MB
      • + *
    • 128MB - 512MB to 16MB
      • + *
    • 512MB - 1.5 GB to 128MB
      • + *
    • Values > 1.5GB return {@link Integer#MAX_VALUE}
      • + *
    + *

    + * @author Brett Okken + */ +public final class CachingBufferPool implements BufferPool { + + private static interface IntFunction { + public E create(int size); + } + + private static final IntFunction ARRAY_FUNCTION = new IntFunction() { + @Override + public byte[] create(int size) { + return new byte[size]; + } + }; + + private static final IntFunction DBB_FUNCTION = new IntFunction() { + @Override + public ByteBuffer create(int size) { + return ByteBuffer.allocateDirect(size); + } + }; + + private static final CachingBufferPool INSTANCE = new CachingBufferPool(); + + private final ConcurrentMap>> bytes = new ConcurrentHashMap<>(); + private final ConcurrentMap>> buffers = new ConcurrentHashMap<>(); + + private CachingBufferPool() { + } + + /** + * Returns instance of {@link CachingBufferPool} for using cached buffers. + * @return instance of {@link CachingBufferPool} for using cached buffers. + */ + public static BufferPool getInstance() { + return INSTANCE; + } + + /** + * {@inheritDoc} + */ + @Override + public byte[] allocateArray(int size) { + if (size <= 0) { + throw new IllegalArgumentException("size is invalid: " + size); + } + + return getOrCreate(size, bytes, ARRAY_FUNCTION); + } + + /** + * {@inheritDoc} + */ + @Override + public void releaseArray(byte[] buffer) { + if (buffer == null) { + throw new IllegalArgumentException("buffer is null"); + } + returnValue(buffer, buffer.length, bytes); + } + + /** + * {@inheritDoc} + */ + @Override + public ByteBuffer allocateDirect(int size) { + if (size <= 0) { + throw new IllegalArgumentException("size is invalid: " + size); + } + + return getOrCreate(size, buffers, DBB_FUNCTION); + } + + /** + * {@inheritDoc} + */ + @Override + public void releaseDirect(ByteBuffer buffer) { + if (buffer == null) { + throw new IllegalArgumentException("buffer is null"); + } + buffer.clear(); + returnValue(buffer, buffer.capacity(), buffers); + } + + private static E getOrCreate(final int size, final ConcurrentMap>> map, final IntFunction creator) { + assert size > 0; + final int adjustedSize = adjustSize(size); + final ConcurrentLinkedDeque> queue = optimisticGetEntry(adjustedSize, map); + SoftReference entry; + while ((entry = queue.pollFirst()) != null) { + final E val = entry.get(); + if (val != null) { + return val; + } + } + + return creator.create(adjustedSize); + } + + /* + * This is package scope to allow direct unit testing. + */ + static int adjustSize(int size) { + assert size > 0; + + switch (Integer.numberOfLeadingZeros(size)) { + case 1: // 1GB - 2GB + case 2: // 512MB + //if 512MB - 1.5 GB round to nearest 128 MB (2^27), else Integer.MAX_VALUE + return size <= 0x6000_0000 ? roundToPowers(size, 27) : Integer.MAX_VALUE; + case 3: //256MB + case 4: //128MB + //if 128MB - 512MB, round to nearest 16 MB + return roundToPowers(size, 24); + case 5: // 64MB + case 6: // 32MB + case 7: // 16MB + //if 16MB - 128MB, round to nearest 4MB + return roundToPowers(size, 22); + case 8: // 8MB + case 9: // 4MB + case 10: // 2MB + //if 2MB - 16MB, round to nearest 512KB + return roundToPowers(size, 19); + case 11: // 1MB + case 12: //512KB + //if 512KB - 2MB, round to nearest 128KB + return roundToPowers(size, 17); + case 13: //256KB + case 14: //128KB + case 15: // 64KB + case 16: // 32KB + //if 32KB to 512KB, round to nearest 16KB + return roundToPowers(size, 14); + case 17: // 16KB + case 18: // 8KB + case 19: // 4KB + // if 4KB - 32KB, round to nearest 2KB + return roundToPowers(size, 11); + default: + return 4 * 1024; + } + } + + private static int roundToPowers(int number, int bits) { + final int mask = (0x7FFF_FFFF >> bits) << bits; + final int floor = number & mask; + return floor == number ? number : floor + (1 << bits); + } + + private static ConcurrentLinkedDeque> optimisticGetEntry(Integer key, ConcurrentMap>> map) { + ConcurrentLinkedDeque> val = map.get(key); + if (val == null) { + map.putIfAbsent(key, new ConcurrentLinkedDeque>()); + val = map.get(key); + } + return val; + } + + private static void returnValue(E value, Integer size, ConcurrentMap>> map) { + final ConcurrentLinkedDeque> queue = map.get(size); + //no queue will exist if buffer was not originally obtained from this class + if (queue != null) { + //push this value onto deque first so that concurrent request can use it + queue.addFirst(new SoftReference(value)); + + //purge oldest entries have lost references + SoftReference entry; + boolean lastEmpty = true; + while(lastEmpty && (entry = queue.peekLast()) != null) { + if (entry.get() == null) { + queue.removeLastOccurrence(entry); + } else { + lastEmpty = false; + } + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return "CachingBufferPool [bytes=" + this.bytes + ", buffers=" + this.buffers + "]"; + } +} + diff --git a/src/main/java/org/xerial/snappy/pool/DefaultPoolFactory.java b/src/main/java/org/xerial/snappy/pool/DefaultPoolFactory.java new file mode 100644 index 00000000..fc9447c1 --- /dev/null +++ b/src/main/java/org/xerial/snappy/pool/DefaultPoolFactory.java @@ -0,0 +1,37 @@ +package org.xerial.snappy.pool; + +/** + * Manages implementation of {@link BufferPool} to use by default. Setting the system property {@link #DISABLE_CACHING_PROPERTY} to {@code true} will + * cause the {@link QuiescentBufferPool} to be used by default. Otherwise, {@link CachingBufferPool} will be used by default. + * {@link #setDefaultPool(BufferPool)} can be used to explicitly control the implementation to use. + */ +public final class DefaultPoolFactory { + + /** + * Name of system property to disable use of {@link CachingBufferPool} by default. + */ + public static final String DISABLE_CACHING_PROPERTY = "org.xerial.snappy.pool.disable"; + + private static volatile BufferPool defaultPool = "true".equalsIgnoreCase(System.getProperty(DISABLE_CACHING_PROPERTY)) + ? QuiescentBufferPool.getInstance() + : CachingBufferPool.getInstance(); + + /** + * @return The default instance to use. + */ + public static BufferPool getDefaultPool() { + return defaultPool; + } + + /** + * Sets the default instance to use. + * @param pool The default instance to use. Must not be {@code null}. + * @see #getDefaultPool() + */ + public static void setDefaultPool(BufferPool pool) { + if (pool == null) { + throw new IllegalArgumentException("pool is null"); + } + defaultPool = pool; + } +} diff --git a/src/main/java/org/xerial/snappy/pool/DirectByteBuffers.java b/src/main/java/org/xerial/snappy/pool/DirectByteBuffers.java new file mode 100644 index 00000000..26b84110 --- /dev/null +++ b/src/main/java/org/xerial/snappy/pool/DirectByteBuffers.java @@ -0,0 +1,154 @@ +package org.xerial.snappy.pool; + +import static java.lang.invoke.MethodHandles.constant; +import static java.lang.invoke.MethodHandles.dropArguments; +import static java.lang.invoke.MethodHandles.filterReturnValue; +import static java.lang.invoke.MethodHandles.guardWithTest; +import static java.lang.invoke.MethodHandles.lookup; +import static java.lang.invoke.MethodType.methodType; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedExceptionAction; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility to facilitate disposing of direct byte buffer instances. + */ +final class DirectByteBuffers { + + /** + * Sun specific mechanisms to clean up resources associated with direct byte buffers. + */ + @SuppressWarnings("unchecked") + static final Class DIRECT_BUFFER_CLAZZ = (Class) lookupClassQuietly("java.nio.DirectByteBuffer"); + + static final MethodHandle CLEAN_HANDLE; + + static { + // this approach is based off that used by apache lucene and documented here: https://issues.apache.org/jira/browse/LUCENE-6989 + // and https://github.com/apache/lucene-solr/blob/7e03427fa14a024ce257babcb8362d2451941e21/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java + MethodHandle cleanHandle = null; + try { + @SuppressWarnings("removal") // AccessController is deprecated for removal in Java 17+ + final PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + + @Override + public MethodHandle run() throws Exception { + MethodHandle handle = null; + if (DIRECT_BUFFER_CLAZZ != null) { + final Lookup lookup = lookup(); + + try { + // sun.misc.Unsafe unmapping (Java 9+) + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + // first check if Unsafe has the right method, otherwise we can give up + // without doing any security critical stuff: + final MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", methodType(void.class, ByteBuffer.class)); + // fetch the unsafe instance and bind it to the virtual MH: + final Field f = unsafeClass.getDeclaredField("theUnsafe"); + f.setAccessible(true); + final Object theUnsafe = f.get(null); + handle = unmapper.bindTo(theUnsafe); + } catch (Exception e) { + Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "unable to use java 9 Unsafe.invokeCleaner", e); + + // sun.misc.Cleaner unmapping (Java 8 and older) + final Method m = DIRECT_BUFFER_CLAZZ.getMethod("cleaner"); + m.setAccessible(true); + final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); + final Class cleanerClass = directBufferCleanerMethod.type().returnType(); + + /* + * "Compile" a MethodHandle that basically is equivalent to the following code: + * void unmapper(ByteBuffer byteBuffer) + * { + * sun.misc.Cleaner cleaner = ((java.nio.DirectByteBuffer) byteBuffer).cleaner(); + * if (nonNull(cleaner)) + * { + * cleaner.clean(); + * } + * else + * { + * // the noop is needed because MethodHandles#guardWithTest always needs ELSE + * noop(cleaner); + * } + * } + */ + final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); + final MethodHandle nonNullTest = lookup.findStatic(DirectByteBuffers.class, "nonNull", methodType(boolean.class, Object.class)).asType(methodType(boolean.class, cleanerClass)); + final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); + handle = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)).asType(methodType(void.class, ByteBuffer.class)); + } + } + + return handle; + } + }; + + @SuppressWarnings("removal") // AccessController is deprecated for removal in Java 17+ + MethodHandle temp = AccessController.doPrivileged(action); + cleanHandle = temp; + + } catch (Throwable t) { + Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t); + } + CLEAN_HANDLE = cleanHandle; + } + + + private static Class lookupClassQuietly(String name) + { + try { + return DirectByteBuffers.class.getClassLoader().loadClass(name); + } + catch (Throwable t) { + Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t); + } + + return null; + } + + + static boolean nonNull(Object o) { + return o != null; + } + + /** + * Provides jvm implementation specific operation to aggressively release resources associated with buffer. + * + * @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}. + */ + @SuppressWarnings("removal") // AccessController is deprecated for removal in Java 17+ + public static void releaseDirectByteBuffer(final ByteBuffer buffer) + { + assert buffer != null && buffer.isDirect(); + + if (CLEAN_HANDLE != null && DIRECT_BUFFER_CLAZZ.isInstance(buffer)) { + try { + final PrivilegedExceptionAction pea = new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + try { + CLEAN_HANDLE.invokeExact(buffer); + } catch (Exception e) { + throw e; + } catch (Throwable t) { + //this will be an error + throw new RuntimeException(t); + } + return null; + } + }; + AccessController.doPrivileged(pea); + } catch (Throwable t) { + Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t); + } + } + } +} diff --git a/src/main/java/org/xerial/snappy/pool/QuiescentBufferPool.java b/src/main/java/org/xerial/snappy/pool/QuiescentBufferPool.java new file mode 100644 index 00000000..8a5c4896 --- /dev/null +++ b/src/main/java/org/xerial/snappy/pool/QuiescentBufferPool.java @@ -0,0 +1,55 @@ +package org.xerial.snappy.pool; + +import java.nio.ByteBuffer; + +/** + * A {@link BufferPool} implementation which does no pooling. New instances will be created for each call to allocate. + * @author Brett Okken + */ +public final class QuiescentBufferPool implements BufferPool { + + private static final QuiescentBufferPool INSTANCE = new QuiescentBufferPool(); + + private QuiescentBufferPool() { + } + + /** + * @return Instance of {@link BufferPool} which does no caching/reuse of instances. + */ + public static BufferPool getInstance() { + return INSTANCE; + } + + /** + * Creates a new {@code byte[]} of size. + */ + @Override + public byte[] allocateArray(int size) { + return new byte[size]; + } + + /** + * Does nothing. + */ + @Override + public void releaseArray(byte[] buffer) { + } + + /** + * {@link ByteBuffer#allocateDirect(int) Allocates} a direct {@link ByteBuffer} of size. + */ + @Override + public ByteBuffer allocateDirect(int size) { + return ByteBuffer.allocateDirect(size); + } + + /** + * Aggressively releases native resources associated with buffer. + */ + @Override + public void releaseDirect(ByteBuffer buffer) { + assert buffer != null && buffer.isDirect(); + DirectByteBuffers.releaseDirectByteBuffer(buffer); + } + +} diff --git a/src/main/resources/META-INF/native-image/org.xerial.snappy/snappy-java/resource-config.json b/src/main/resources/META-INF/native-image/org.xerial.snappy/snappy-java/resource-config.json new file mode 100644 index 00000000..00446fc0 --- /dev/null +++ b/src/main/resources/META-INF/native-image/org.xerial.snappy/snappy-java/resource-config.json @@ -0,0 +1,11 @@ +{ + "resources": { + "includes": [ + {"pattern":".*libsnappyjava.dylib"}, + {"pattern":".*libsnappyjava.so"}, + {"pattern":".*snappyjava.dll"}, + {"pattern":".*libsnappyjava.a"} + ] + }, + "bundles": [] +} \ No newline at end of file diff --git a/src/main/resources/org/xerial/snappy/VERSION b/src/main/resources/org/xerial/snappy/VERSION index 244b3cda..b9fcc513 100755 --- a/src/main/resources/org/xerial/snappy/VERSION +++ b/src/main/resources/org/xerial/snappy/VERSION @@ -1,2 +1,2 @@ -SNAPPY_VERSION=1.1.7 -BITSHUFFLE_VERSION=0.3.2 +SNAPPY_VERSION=1.1.10 +BITSHUFFLE_VERSION=0.3.4 diff --git a/src/main/resources/org/xerial/snappy/native/FreeBSD/x86/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/FreeBSD/x86/libsnappyjava.so new file mode 100755 index 00000000..0a5225d6 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/FreeBSD/x86/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/FreeBSD/x86_64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/FreeBSD/x86_64/libsnappyjava.so index 338088cd..fae3117a 100755 Binary files a/src/main/resources/org/xerial/snappy/native/FreeBSD/x86_64/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/FreeBSD/x86_64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so index 5729aeee..e6089826 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/android-aarch64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/android-aarch64/libsnappyjava.so new file mode 100755 index 00000000..8403b187 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Linux/android-aarch64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/android-arm/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/android-arm/libsnappyjava.so index fa01fcdc..f40046de 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/android-arm/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/android-arm/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so index ceaeccbb..670e810c 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/armv6/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/armv6/libsnappyjava.so index 7291b064..42fa69e8 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/armv6/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/armv6/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/armv7/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/armv7/libsnappyjava.so index b4a84672..3550dd3e 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/armv7/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/armv7/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/loongarch64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/loongarch64/libsnappyjava.so new file mode 100644 index 00000000..534a9d0b Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Linux/loongarch64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so index e8c4fb14..cd0f6a39 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/riscv64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/riscv64/libsnappyjava.so new file mode 100755 index 00000000..3489c588 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Linux/riscv64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/s390x/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/s390x/libsnappyjava.so index a1d301e9..8d166bf8 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/s390x/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/s390x/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so index de9f9f81..7aa2a42d 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/x86_64-musl/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/x86_64-musl/libsnappyjava.so new file mode 100755 index 00000000..20ecf776 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Linux/x86_64-musl/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so index 69998c9a..ccb82feb 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Mac/aarch64/libsnappyjava.dylib b/src/main/resources/org/xerial/snappy/native/Mac/aarch64/libsnappyjava.dylib new file mode 100755 index 00000000..35a8fc8a Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Mac/aarch64/libsnappyjava.dylib differ diff --git a/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.dylib b/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.dylib new file mode 100755 index 00000000..a79237ee Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.dylib differ diff --git a/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib b/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib deleted file mode 100755 index a7e5558a..00000000 Binary files a/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib and /dev/null differ diff --git a/src/main/resources/org/xerial/snappy/native/Windows/aarch64/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/aarch64/snappyjava.dll new file mode 100755 index 00000000..95de0420 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Windows/aarch64/snappyjava.dll differ diff --git a/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll index e97d8b23..ffa1c79a 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll and b/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll differ diff --git a/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll index dcf63995..35bb40d7 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll and b/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll differ diff --git a/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java b/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java index b03ec870..e428bf98 100644 --- a/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java @@ -9,7 +9,6 @@ import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; -import static org.xerial.snappy.SnappyFramed.maskedCrc32c; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -418,4 +417,9 @@ private byte[] getRandom(double compressionRatio, int length) assertEquals(random.length, length); return random; } + + public static int maskedCrc32c(byte[] data) + { + return SnappyFramed.maskedCrc32c(new PureJavaCrc32C(), data, 0, data.length); + } } diff --git a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java index ec493e10..e9c66bf9 100644 --- a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java @@ -2,12 +2,12 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.SystemUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.SnappyCodec; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import java.io.*; @@ -24,13 +24,13 @@ public static void loadHadoopNativeLibrary() throws Exception { final String libResourceFolder; Map libraryNames = new LinkedHashMap<>(); - if (SystemUtils.IS_OS_LINUX) { + if (OSInfo.getOSName() == "Linux") { libResourceFolder = "/lib/Linux"; libraryNames.put("libhadoop.so", "libhadoop.so"); // certain Linux systems need these shared library be copied before the JVM started, see build.sbt libraryNames.put("libsnappy.so", "libsnappy.so"); libraryNames.put("libsnappy.so.1", "libsnappy.so"); - } else if (SystemUtils.IS_OS_MAC_OSX) { + } else if (OSInfo.getOSName() == "Mac") { libResourceFolder = "/lib/MacOSX"; libraryNames.put("libhadoop.dylib", "libhadoop.dylib"); libraryNames.put("libsnappy.dylib", "libsnappy.dylib"); @@ -76,6 +76,7 @@ public static void cleanUpLibraryFolder() FileUtils.deleteQuietly(tempNativeLibFolder); } + @Ignore("This test doesn't work with openjdk11 in GitHub Action") @Test public void testXerialCompressionHadoopDecompressionCodec() throws Exception { diff --git a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java index f2b7774a..36ef552e 100755 --- a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java @@ -34,6 +34,7 @@ import java.nio.ByteOrder; import org.junit.Test; +import org.junit.Assert; import org.xerial.snappy.buffer.BufferAllocatorFactory; import org.xerial.snappy.buffer.CachedBufferAllocator; import org.xerial.snappy.buffer.DefaultBufferAllocator; @@ -106,6 +107,17 @@ public void bufferSize() is.close(); } + @Test(expected = IllegalArgumentException.class) + public void invalidBlockSize() + throws Exception + { + // We rely on catch below, if there is no error this test will pass + // This can be done better with Assertions.assertThrows + Boolean exceptionThrown = false; + ByteArrayOutputStream b = new ByteArrayOutputStream(); + SnappyOutputStream os = new SnappyOutputStream(b, 1024 * 1024 * 1024); + } + @Test public void smallWrites() throws Exception @@ -167,6 +179,9 @@ public void batchingOfWritesShouldNotAffectCompressedDataSize() // compression quality: if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) assertEquals(90992, expectedCompressedData.length); + else if(OSInfo.getArchName() == "aarch64") + // Arm has a better compression ratio + assertEquals(91051, expectedCompressedData.length); else assertEquals(91080, expectedCompressedData.length); // The chunk size should not affect the size of the compressed output: diff --git a/src/test/java/org/xerial/snappy/SnappyTest.java b/src/test/java/org/xerial/snappy/SnappyTest.java index 18b39e92..52c3a005 100755 --- a/src/test/java/org/xerial/snappy/SnappyTest.java +++ b/src/test/java/org/xerial/snappy/SnappyTest.java @@ -19,16 +19,18 @@ // SnappyTest.java // Since: 2011/03/30 // -// $URL$ +// $URL$ // $Author$ //-------------------------------------- package org.xerial.snappy; import static org.junit.Assert.*; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; +import org.junit.Assume; import org.junit.Assert; import org.junit.Test; import org.xerial.util.log.Logger; @@ -329,4 +331,205 @@ public void isValidCompressedData() _logger.debug(e); } } + + + /* + Tests happy cases for SnappyInputStream.read method + - {0} + */ + @Test + public void isValidChunkLengthForSnappyInputStreamIn() + throws Exception { + byte[] data = {0}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + in.read(out); + } + + /* + Tests sad cases for SnappyInputStream.read method + - Expects a java.lang.NegativeArraySizeException catched into a SnappyError + - {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff} + */ + @Test(expected = SnappyError.class) + public void isInvalidChunkLengthForSnappyInputStreamInNegative() + throws Exception { + byte[] data = {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + in.read(out); + } + + /* + Tests sad cases for SnappyInputStream.read method + - Expects a java.lang.OutOfMemoryError + - {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff} + */ + @Test(expected = SnappyError.class) + public void isInvalidChunkLengthForSnappyInputStreamOutOfMemory() + throws Exception { + byte[] data = {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + try { + in.read(out); + } catch (Exception ignored) { + // Exception here will be catched + // But OutOfMemoryError will not be caught, and will still be thrown + } + } + + /* + Tests sad cases for SnappyInputStream.read method + - Expects a failed to compress exception due to upper bounds chunk size + - {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff} + */ + @Test + public void isInvalidChunkLengthForSnappyInputStream() + throws Exception { + byte[] data = {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + try { + in.read(out); + } catch (SnappyError error) { + Assert.assertEquals(error.errorCode, SnappyErrorCode.FAILED_TO_UNCOMPRESS); + } + } + + /* + Tests happy cases for BitShuffle.shuffle method + - double: 0, 10 + - float: 0, 10 + - int: 0, 10 + - long: 0, 10 + - short: 0, 10 + */ + @Test + public void isValidArrayInputLength() + throws Exception { + byte[] a = Snappy.compress(new char[0]); + byte[] b = Snappy.compress(new double[0]); + byte[] c = Snappy.compress(new float[0]); + byte[] d = Snappy.compress(new int[0]); + byte[] e = Snappy.compress(new long[0]); + byte[] f = Snappy.compress(new short[0]); + byte[] g = Snappy.compress(new char[10]); + byte[] h = Snappy.compress(new double[10]); + byte[] i = Snappy.compress(new float[10]); + byte[] j = Snappy.compress(new int[10]); + byte[] k = Snappy.compress(new long[10]); + byte[] l = Snappy.compress(new short[10]); + } + + /* + Tests sad cases for Snappy.compress + - Allocate a buffer whose byte size will be a bit larger than Integer.MAX_VALUE + - char + - double + - float + - int + - long + - short + */ + @Test(expected = SnappyError.class) + public void isTooLargeDoubleArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new double[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeCharArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new char[Integer.MAX_VALUE / 2 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeFloatArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new float[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeIntArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new int[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeLongArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new long[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeShortArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new short[Integer.MAX_VALUE / 2 + 1]); + } + + /* + Tests happy cases for Snappy.compress + - char: 0, 10 + */ + @Test + public void isValidArrayInputLengthForBitShuffleShuffle() + throws Exception + { + byte[] b = BitShuffle.shuffle(new double[0]); + byte[] c = BitShuffle.shuffle(new float[0]); + byte[] d = BitShuffle.shuffle(new int[0]); + byte[] e = BitShuffle.shuffle(new long[0]); + byte[] f = BitShuffle.shuffle(new short[0]); + byte[] n = BitShuffle.shuffle(new double[10]); + byte[] o = BitShuffle.shuffle(new float[10]); + byte[] p = BitShuffle.shuffle(new int[10]); + byte[] q = BitShuffle.shuffle(new long[10]); + byte[] r = BitShuffle.shuffle(new short[10]); + } + + /* + Tests sad cases for BitShuffle.shuffle method + - Allocate a buffer whose byte size will be a bit larger than Integer.MAX_VALUE + - double: 8 + - float: 4 + - int: 4 + - long: 8 + - short: 2 + */ + @Test(expected = SnappyError.class) + public void isTooLargeDoubleArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new double[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeFloatArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new float[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeIntArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new float[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeLongArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new long[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeShortArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new short[Integer.MAX_VALUE / 2 + 1]); + } + + private void assumingCIIsFalse() { + if (System.getenv("CI") == null) + return; + Assume.assumeFalse("Skipped on CI", System.getenv("CI").equals("true")); + } } diff --git a/src/test/java/org/xerial/snappy/pool/CachingBufferPoolTest.java b/src/test/java/org/xerial/snappy/pool/CachingBufferPoolTest.java new file mode 100644 index 00000000..ac25bc98 --- /dev/null +++ b/src/test/java/org/xerial/snappy/pool/CachingBufferPoolTest.java @@ -0,0 +1,188 @@ +package org.xerial.snappy.pool; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.lang.ref.Reference; +import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + + +public class CachingBufferPoolTest { + + private static final int LIST_COUNT = 2048; + + @Test + public void testAdjustSize() { + assertEquals(4 * 1024, CachingBufferPool.adjustSize(2)); + assertEquals(4 * 1024, CachingBufferPool.adjustSize(1023)); + assertEquals(4 * 1024, CachingBufferPool.adjustSize(1024)); + assertEquals(4 * 1024, CachingBufferPool.adjustSize(1025)); + assertEquals(4 * 1024, CachingBufferPool.adjustSize(4 * 1024)); + assertEquals((4 + 2) * 1024, CachingBufferPool.adjustSize((4 * 1024) + 1)); + assertEquals(6 * 1024, CachingBufferPool.adjustSize(5 * 1024)); + assertEquals(6 * 1024, CachingBufferPool.adjustSize((5 * 1024) + 1)); + + assertEquals(32 * 1024, CachingBufferPool.adjustSize(32 * 1024)); + assertEquals((32 + 16) * 1024, CachingBufferPool.adjustSize((32 * 1024) + 1)); + + assertEquals(2 * 1024 * 1024, CachingBufferPool.adjustSize(2 * 1024 * 1024)); + assertEquals(((2 * 1024) + 512) * 1024, CachingBufferPool.adjustSize((2 * 1024 * 1024) + 1)); + + assertEquals(16 * 1024 * 1024, CachingBufferPool.adjustSize(16 * 1024 * 1024)); + assertEquals((16 + 4) * 1024 * 1024, CachingBufferPool.adjustSize((16 * 1024 * 1024) + 1)); + + assertEquals(128 * 1024 * 1024, CachingBufferPool.adjustSize(128 * 1024 * 1024)); + assertEquals((128 + 16) * 1024 * 1024, CachingBufferPool.adjustSize((128 * 1024 * 1024) + 1)); + + assertEquals(512 * 1024 * 1024, CachingBufferPool.adjustSize(512 * 1024 * 1024)); + assertEquals((512 + 128) * 1024 * 1024, CachingBufferPool.adjustSize((512 * 1024 * 1024) + 1)); + assertEquals(0x6000_0000, CachingBufferPool.adjustSize(0x6000_0000)); + assertEquals(0x6000_0000, CachingBufferPool.adjustSize(0x6000_0000 - 1)); + assertEquals(Integer.MAX_VALUE, CachingBufferPool.adjustSize(0x6000_0001)); + assertEquals(Integer.MAX_VALUE, CachingBufferPool.adjustSize(Integer.MAX_VALUE)); + assertEquals(Integer.MAX_VALUE, CachingBufferPool.adjustSize(Integer.MAX_VALUE - 1)); + } + + @Test + public void testDirectByteBuffers() throws Exception { + + BufferPool pool = CachingBufferPool.getInstance(); + + ByteBuffer bb1 = pool.allocateDirect(12 * 1024); + assertNotNull(bb1); + assertEquals(12 * 1024, bb1.limit()); + assertEquals(12 * 1024, bb1.capacity()); + assertEquals(0, bb1.position()); + + ByteBuffer bb2 = pool.allocateDirect(12 * 1024); + assertNotNull(bb2); + assertEquals(12 * 1024, bb2.limit()); + assertEquals(12 * 1024, bb2.capacity()); + assertEquals(0, bb2.position()); + + assertNotSame(bb1, bb2); + + bb2.position(18); + pool.releaseDirect(bb2); + + ByteBuffer bb3 = pool.allocateDirect(12 * 1024); + assertNotNull(bb3); + assertEquals(12 * 1024, bb3.limit()); + assertEquals(12 * 1024, bb3.capacity()); + assertEquals(0, bb3.position()); + + assertNotSame(bb1, bb2); + assertSame(bb2, bb3); + + pool.releaseDirect(bb1); + + ByteBuffer bb4 = pool.allocateDirect((12 * 1024) - 1); + assertNotNull(bb4); + assertEquals(12 * 1024, bb4.limit()); + assertEquals(12 * 1024, bb4.capacity()); + assertEquals(0, bb4.position()); + + assertSame(bb1, bb4); + } + + @Test + public void testArrays() throws Exception { + + BufferPool pool = CachingBufferPool.getInstance(); + + byte[] bb1 = pool.allocateArray(12 * 1024); + assertNotNull(bb1); + assertEquals(12 * 1024, bb1.length); + + byte[] bb2 = pool.allocateArray(12 * 1024); + assertNotNull(bb2); + assertEquals(12 * 1024, bb2.length); + + assertNotSame(bb1, bb2); + + pool.releaseArray(bb2); + + byte[] bb3 = pool.allocateArray(12 * 1024); + assertNotNull(bb3); + assertEquals(12 * 1024, bb3.length); + + assertNotSame(bb1, bb2); + assertSame(bb2, bb3); + + pool.releaseArray(bb1); + + byte[] bb4 = pool.allocateArray((12 * 1024) - 1); + assertNotNull(bb4); + assertEquals(12 * 1024, bb4.length); + + assertSame(bb1, bb4); + } + + @Test + public void testSoftReferences() { + + BufferPool pool = CachingBufferPool.getInstance(); + byte[] bb1 = pool.allocateArray(8 * 1024); + Reference ref = new WeakReference(bb1); + bb1[0] = 123; + bb1[8000] = -74; + int bb1HC = System.identityHashCode(bb1); + + pool.releaseArray(bb1); + + byte[] bb1_copy = pool.allocateArray(8 * 1024); + assertSame(bb1, bb1_copy); + assertEquals(123, bb1_copy[0]); + assertEquals(-74, bb1_copy[8000]); + assertEquals(bb1HC, System.identityHashCode(bb1_copy)); + + //release back into pool (again) + pool.releaseArray(bb1); + + //release strong references + bb1_copy = null; + bb1 = null; + assertNotNull(ref.get()); + + //force an OOME to for SoftReferences to be collected + try { + List vals = forceOOMEGC(LIST_COUNT); + assertTrue("count: " + vals.size(), vals.size() < LIST_COUNT); + } catch (OutOfMemoryError e) { + // + } + + //assert that our test reference has been cleared + assertNull(ref.get()); + + //get another value from the pool + byte[] bb2 = pool.allocateArray(8 * 1024); + //assert that it is indeed a new value, and not same from previous + assertNotEquals(123, bb2[0]); + assertNotEquals(-74, bb2[8000]); + assertNotEquals(bb1HC, System.identityHashCode(bb2)); + } + + private static List forceOOMEGC(int count) { + final List vals = new ArrayList<>(count); + + try { + for (int i = 0; i < count; ++i) { + vals.add(new byte[10 * 1024 * 1024]); + } + } catch (Error e) { + + } + return vals; + } +} diff --git a/src/test/resources/integration/SnappyIntegrationTest.java b/src/test/resources/integration/SnappyIntegrationTest.java new file mode 100644 index 00000000..618c0bf5 --- /dev/null +++ b/src/test/resources/integration/SnappyIntegrationTest.java @@ -0,0 +1,32 @@ +import org.xerial.snappy.Snappy; + +/** + * Simple integration test to verify snappy-java works in a separate JVM. + * This is compiled and run as a standalone program to test the JAR manifest + * and native library loading on different JDK versions. + */ +public class SnappyIntegrationTest { + public static void main(String[] args) throws Exception { + String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of " + + "Snappy, a fast compresser/decompresser."; + + // Test compression + byte[] compressed = Snappy.compress(input.getBytes("UTF-8")); + System.out.println("Compressed " + input.length() + " bytes to " + compressed.length + " bytes"); + + // Test decompression + byte[] uncompressed = Snappy.uncompress(compressed); + String result = new String(uncompressed, "UTF-8"); + + // Verify result matches input + if (!result.equals(input)) { + System.err.println("ERROR: Decompressed data does not match input!"); + System.err.println("Expected: " + input); + System.err.println("Got: " + result); + System.exit(1); + } + + System.out.println("SUCCESS: " + result); + System.exit(0); + } +} diff --git a/src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala b/src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala deleted file mode 100644 index 2a1c2a9b..00000000 --- a/src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala +++ /dev/null @@ -1,44 +0,0 @@ -package org.xerial.snappy - -import java.io.{ByteArrayOutputStream, ByteArrayInputStream} - -import scala.util.Random - -/** - * - */ -class SnappyPerformanceTest extends SnappySpec { - - lazy val data = { - val a = new Array[Byte](128 * 1024 * 1024) - - for (i <- (0 until a.length).par) { - a(i) = Math.sin(i * 0.01).toByte - } - a - } - - "SnappyOutputStream" should { - - "improve output performance" taggedAs ("out") in { - - val input = data - - time("compression", repeat = 100) { - // 0.037 sec. => 0.026 - block("default") { - val out = new ByteArrayOutputStream() - val sout = new SnappyOutputStream(out) - sout.write(input) - sout.close() - out.close() - } - - } - - //info(f"compressed size: ${compressed.length}%,d, input: ${data.length}%,d") - } - - } - -} diff --git a/src/test/scala/org/xerial/snappy/SnappySpec.scala b/src/test/scala/org/xerial/snappy/SnappySpec.scala deleted file mode 100644 index f4c4d72b..00000000 --- a/src/test/scala/org/xerial/snappy/SnappySpec.scala +++ /dev/null @@ -1,15 +0,0 @@ -package org.xerial.snappy - -import org.scalatest._ -import wvlet.log.LogFormatter.SourceCodeLogFormatter -import wvlet.log.{LogSupport, Logger} -import wvlet.log.io.Timer - -/** - * - */ -trait SnappySpec extends WordSpec with Matchers with GivenWhenThen with OptionValues with BeforeAndAfter with Timer with LogSupport { - Logger.setDefaultFormatter(SourceCodeLogFormatter) - - implicit def toTag(s: String): Tag = Tag(s) -} diff --git a/version.sbt b/version.sbt deleted file mode 100644 index 672b684c..00000000 --- a/version.sbt +++ /dev/null @@ -1 +0,0 @@ -version in ThisBuild := "1.1.7.2-SNAPSHOT"