diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..76acfc00 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,23 @@ +# Ensure all text files use LF line endings in the repository +* text=auto + +# Explicitly set LF for specific files that are checked in tests +*.go text eol=lf +*.md text eol=lf +*.sh text eol=lf +*.py text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.txt text eol=lf + +# Binary files +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.so binary +*.pyd binary +*.dll binary +*.exe binary +*.pyc binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1b0b9e0..af0017d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + branches: [ '**' ] env: TAGS: "-tags=ci" @@ -13,71 +13,60 @@ env: # GOPY_TRAVIS_CI is set GOPY_TRAVIS_CI: 1 GOTRACEBACK: crash - PYPYVERSION: "v7.1.1" GO111MODULE: auto jobs: build: - name: Build + name: Build (${{ matrix.platform }}, Go ${{ matrix.go-version }}, Python ${{ matrix.python-version }}) strategy: + fail-fast: false matrix: - go-version: [1.16.x, 1.15.x] - platform: [ubuntu-latest] - #platform: [ubuntu-latest, macos-latest, windows-latest] + # TODO: Consider official support matrix (OS and Go versions) and adjust this matrix accordingly + go-version: [1.25.x, 1.24.x, 1.23.x, 1.22.x] + platform: [ubuntu-latest, windows-latest, macos-15] + python-version: ['3.11', '3.12'] runs-on: ${{ matrix.platform }} steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - - - name: Cache-Go - uses: actions/cache@v1 - with: - path: | - ~/go/pkg/mod # Module download cache - ~/.cache/go-build # Build cache (Linux) - ~/Library/Caches/go-build # Build cache (Mac) - '%LocalAppData%\go-build' # Build cache (Windows) - - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - - - name: Checkout code - uses: actions/checkout@v2 + cache: true - name: Install Linux packages if: matrix.platform == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install curl libffi-dev python-cffi python3-cffi python3-pip - # pypy3 isn't packaged in ubuntu yet. - TEMPDIR=$(mktemp -d) - curl -L https://downloads.python.org/pypy/pypy2.7-${PYPYVERSION}-linux64.tar.bz2 --output $TEMPDIR/pypy2.tar.bz2 - curl -L https://downloads.python.org/pypy/pypy3.6-${PYPYVERSION}-linux64.tar.bz2 --output $TEMPDIR/pypy3.tar.bz2 - tar xf $TEMPDIR/pypy2.tar.bz2 -C $TEMPDIR - tar xf $TEMPDIR/pypy3.tar.bz2 -C $TEMPDIR - sudo ln -s $TEMPDIR/pypy2.7-$PYPYVERSION-linux64/bin/pypy /usr/local/bin/pypy - sudo ln -s $TEMPDIR/pypy3.6-$PYPYVERSION-linux64/bin/pypy3 /usr/local/bin/pypy3 - # install pip (for pypy, python2) - curl -L https://bootstrap.pypa.io/pip/2.7/get-pip.py --output ${TEMPDIR}/get-pip2.py - python2 ${TEMPDIR}/get-pip2.py - # curl -L https://bootstrap.pypa.io/get-pip.py --output ${TEMPDIR}/get-pip.py - # pypy ${TEMPDIR}/get-pip.py - # pypy3 ${TEMPDIR}/get-pip.py - + sudo apt-get install curl libffi-dev python3-cffi python3-pip # install pybindgen - python2 -m pip install --user -U pybindgen python3 -m pip install --user -U pybindgen - # pypy -m pip install --user -U pybindgen - # pypy3 -m pip install --user -U pybindgen + # install goimports + go install golang.org/x/tools/cmd/goimports@latest + + - name: Install macOS packages + if: startsWith(matrix.platform, 'macos-') + run: | + python3 -m pip install -U pybindgen + go install golang.org/x/tools/cmd/goimports@latest + - name: Install Windows packages + if: matrix.platform == 'windows-latest' + run: | + # install pybindgen and psutil (for memory tracking in tests) + python -m pip install -U pybindgen psutil # install goimports - go get golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest + - - name: Build-Linux if: matrix.platform == 'ubuntu-latest' run: | @@ -86,6 +75,24 @@ jobs: if: matrix.platform == 'ubuntu-latest' run: | make test + + - name: Build-macOS + if: startsWith(matrix.platform, 'macos-') + run: | + make + - name: Test macOS + if: startsWith(matrix.platform, 'macos-') + run: | + make test + + - name: Build-Windows + if: matrix.platform == 'windows-latest' + run: | + go build -v ./... + - name: Test Windows + if: matrix.platform == 'windows-latest' + run: | + go test -v ./... - name: Upload-Coverage if: matrix.platform == 'ubuntu-latest' - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index a6476240..8b1d6204 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -6,8 +6,9 @@ The `go-python` project (and `gopy`) eagerly accepts contributions from the comm The `go-python` project provides libraries and tools in Go for the Go community to better integrate with Python projects and libraries, and we would like you to join us in improving `go-python`'s quality and scope. -This document is for contributors or those interested in contributing. -Questions about `go-python` and the use of its libraries can be directed to the [go-python](mailto:go-python@googlegroups.com) mailing list. +This document is for contributors or those interested in contributing. + +Questions about `gopy` can be directed to the [the `gopy` discussions area in Github: https://github.com/go-python/gopy/discussions]. ## Contributing @@ -35,7 +36,7 @@ As a rule, we keep all tests OK and try to increase code coverage. ### Suggesting Enhancements If the scope of the enhancement is small, open an issue. -If it is large, such as suggesting a new repository, sub-repository, or interface refactoring, then please start a discussion on [the go-python list](https://groups.google.com/forum/#!forum/go-python). +If it is large, such as suggesting a new repository, sub-repository, or interface refactoring, then please start a discussion using [the `gopy` discussions area in Github: https://github.com/go-python/gopy/discussions]. ### Your First Code Contribution @@ -140,3 +141,5 @@ We use [Go style](https://github.com/golang/go/wiki/CodeReviewComments). This _"Contributing"_ guide has been extracted from the [Gonum](https://gonum.org) project. Its guide is [here](https://github.com/gonum/license/blob/master/CONTRIBUTING.md). + +[the `gopy` discussions area in Github: https://github.com/go-python/gopy/discussions]: https://github.com/go-python/gopy/discussions \ No newline at end of file diff --git a/Makefile b/Makefile index 6055b3e8..604892aa 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,9 @@ GOGET=$(GOCMD) get DIRS=`go list ./...` +PYTHON=python3 +PIP=$(PYTHON) -m pip + all: build build: @@ -40,15 +43,18 @@ mod-update: go get -u ./... go mod tidy -# gopath-update is for GOPATH to get most things updated. -# need to call it in a target executable directory -gopath-update: export GO111MODULE = off -gopath-update: - @echo "GO111MODULE = $(value GO111MODULE)" - go get -u ./... +prereq: + @echo "Installing python prerequisites -- ignore err if already installed:" + - $(PIP) install -r requirements.txt + @echo + @echo "if this fails, you may see errors like this:" + @echo " Undefined symbols for architecture x86_64:" + @echo " _PyInit__gi, referenced from:..." + @echo + # NOTE: MUST update version number here prior to running 'make release' and edit this file! -VERS=v0.4.5 +VERS=v0.4.10 PACKAGE=main GIT_COMMIT=`git rev-parse --short HEAD` VERS_DATE=`date -u +%Y-%m-%d\ %H:%M` diff --git a/README.md b/README.md index 885f33d0..2eec0f17 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ Currently using [pybindgen](https://pybindgen.readthedocs.io/en/latest/tutorial/ ```sh $ python3 -m pip install pybindgen -$ go get golang.org/x/tools/cmd/goimports -$ go get github.com/go-python/gopy +$ go install golang.org/x/tools/cmd/goimports@latest +$ go install github.com/go-python/gopy@latest ``` (This all assumes you have already installed [Go itself](https://golang.org/doc/install), and added `~/go/bin` to your `PATH`). @@ -57,6 +57,16 @@ https://stackoverflow.com/questions/39910730/python3-is-not-recognized-as-an-int If you get a bunch of errors during linking in the build process, set `LIBDIR` or `GOPY_LIBDIR` to path to python libraries, and `LIBRARY` or `GOPY_PYLIB` to name of python library (e.g., python39 for 3.9). +#### Running Tests on Windows + +To run the test suite on Windows, you need to install `psutil` for memory tracking. This is required because Python's built-in `resource` module is [only available on Unix](https://docs.python.org/3/library/resource.html): + +```sh +python -m pip install psutil +``` + +Without `psutil`, tests that check for memory leaks will fail with `ModuleNotFoundError`. + ## Community See the [CONTRIBUTING](https://github.com/go-python/gopy/blob/master/CONTRIBUTE.md) guide for pointers on how to contribute to `gopy`. @@ -269,11 +279,11 @@ To know what features are supported on what backends, please refer to the ## setup.py -Here's an example for how to use `setup.py` with gopy to make an installable package: -https://github.com/natun-ai/labsdk/blob/master/setup.py +Here's an example for how to use `setup.py` with gopy to make an installable package: +https://github.com/raptor-ml/raptor/blob/master/labsdk/setup.py Also, see this for cross-platform build: -https://github.com/natun-ai/labsdk/blob/master/.github/workflows/wheels.yaml +https://github.com/raptor-ml/raptor/blob/master/.github/workflows/labsdk-release.yaml ## Troubleshooting diff --git a/SUPPORT_MATRIX.md b/SUPPORT_MATRIX.md index f23c8f62..8f30be77 100644 --- a/SUPPORT_MATRIX.md +++ b/SUPPORT_MATRIX.md @@ -3,31 +3,33 @@ NOTE: File auto-generated by TestCheckSupportMatrix in main_test.go. Please don't modify manually. -Feature |py2 | py3 ---- | --- | --- -_examples/arrays | yes | yes -_examples/cgo | yes | yes -_examples/consts | yes | yes -_examples/cstrings | yes | yes -_examples/empty | yes | yes -_examples/funcs | yes | yes -_examples/gopygc | yes | yes -_examples/gostrings | yes | yes -_examples/hi | no | yes -_examples/iface | no | yes -_examples/lot | yes | yes -_examples/maps | yes | yes -_examples/named | yes | yes -_examples/osfile | yes | yes -_examples/pkgconflict | yes | yes -_examples/pointers | yes | yes -_examples/pyerrors | yes | yes -_examples/rename | yes | yes -_examples/seqs | yes | yes -_examples/simple | yes | yes -_examples/sliceptr | yes | yes -_examples/slices | yes | yes -_examples/structs | yes | yes -_examples/unicode | no | yes -_examples/variadic | no | yes -_examples/vars | yes | yes +Feature |py3 +--- | --- +_examples/arrays | yes +_examples/cgo | yes +_examples/consts | yes +_examples/cstrings | yes +_examples/empty | yes +_examples/funcs | yes +_examples/gilstring | yes +_examples/gobytes | yes +_examples/gopygc | yes +_examples/gostrings | yes +_examples/hi | yes +_examples/iface | yes +_examples/lot | yes +_examples/maps | yes +_examples/named | yes +_examples/osfile | yes +_examples/pkgconflict | yes +_examples/pointers | yes +_examples/pyerrors | yes +_examples/rename | yes +_examples/seqs | yes +_examples/simple | yes +_examples/sliceptr | yes +_examples/slices | yes +_examples/structs | yes +_examples/unicode | yes +_examples/variadic | yes +_examples/vars | yes diff --git a/_examples/cgo/cgo.go b/_examples/cgo/cgo.go index 98a64a31..4119d47a 100644 --- a/_examples/cgo/cgo.go +++ b/_examples/cgo/cgo.go @@ -9,7 +9,7 @@ package cgo //#include //#include //const char* cpkg_sprintf(const char *str) { -// char *o = (char*)malloc(strlen(str)); +// char *o = (char*)malloc(strlen(str) + 1); // sprintf(o, "%s", str); // return o; //} diff --git a/_examples/cstrings/test.py b/_examples/cstrings/test.py index d7b8a830..74def9ce 100644 --- a/_examples/cstrings/test.py +++ b/_examples/cstrings/test.py @@ -7,7 +7,16 @@ import cstrings import gc -import resource +import sys + +# resource module is Unix-only, not available on Windows +# On Windows, use psutil for memory tracking +if sys.platform == 'win32': + import psutil + HAS_RESOURCE = False +else: + import resource + HAS_RESOURCE = True verbose = False iterations = 10000 @@ -39,7 +48,11 @@ def gofnMap(): def print_memory(s): - m = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if HAS_RESOURCE: + m = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + else: + # psutil returns memory in bytes, convert to KB to match resource module + m = psutil.Process().memory_info().rss // 1024 if verbose: print(s, m) return m @@ -69,17 +82,18 @@ def _run_fn(fn): for fn in [gofnString, gofnStruct, gofnNestedStruct, gofnSlice, gofnMap]: alloced = size * iterations - a = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + a = print_memory("Initial memory:") pass1 = _run_fn(fn) - b = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + b = print_memory("After first pass:") pass2 = _run_fn(fn) - c = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + c = print_memory("After second pass:") if verbose: print(fn.__name__, pass1) print(fn.__name__, pass2) print(fn.__name__, a, b, c) - print(fn.__name__, "leaked: ", (c-b) > (size * iterations)) + leaked = (c-b) > (size * iterations) + print(fn.__name__, "leaked: ", leaked) # bump up the size of each successive test to ensure that leaks # are not absorbed by previous rss growth. diff --git a/_examples/gilstring/gilstring.go b/_examples/gilstring/gilstring.go new file mode 100644 index 00000000..9410c57a --- /dev/null +++ b/_examples/gilstring/gilstring.go @@ -0,0 +1,14 @@ +// Copyright 2026 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gilstring is a regression test for the multi-runtime crash (issue #370). +// It mirrors the exact reproduction from the issue report: a string-returning +// function called alongside an integer function from a second extension in the +// same Python process, which triggers crashes under repeated calls. +package gilstring + +import "fmt" + +// Hello returns a greeting string, mirroring hi.Hello from the issue report. +func Hello(s string) string { return fmt.Sprintf("Hello, %s!", s) } diff --git a/_examples/gilstring/test.py b/_examples/gilstring/test.py new file mode 100644 index 00000000..1aae7b31 --- /dev/null +++ b/_examples/gilstring/test.py @@ -0,0 +1,18 @@ +# Copyright 2026 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +## py2/py3 compat +from __future__ import print_function + +# Regression test for multi-runtime crash (issue #370). +# Exact reproduction from the issue report: two separately-built gopy +# extensions loaded in the same process, with calls interleaved in a loop. +from gilstring.gilstring import Hello +from simple.simple import Add + +for _ in range(5000): + Add(2, 2) + Hello('hi') + +print("OK") diff --git a/_examples/gobytes/gobytes.go b/_examples/gobytes/gobytes.go new file mode 100644 index 00000000..f7721e8a --- /dev/null +++ b/_examples/gobytes/gobytes.go @@ -0,0 +1,33 @@ +// Copyright 2017 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gobytes + +func HashBytes(b []byte) [4]byte { + result := [4]byte{0, 0, 0, 0} + full_blocks := len(b) / 4 + for i := 0; i < full_blocks; i++ { + for j := 0; j < 4; j++ { + result[j] ^= b[4*i+j] + } + } + if full_blocks*4 < len(b) { + for j := 0; j < 4; j++ { + if full_blocks*4+j < len(b) { + result[j] ^= b[full_blocks*4+j] + } else { + result[j] ^= 0x55 + } + } + } + return result +} + +func CreateBytes(len byte) []byte { + res := make([]byte, len) + for i := (byte)(0); i < len; i++ { + res[i] = i + } + return res +} diff --git a/_examples/gobytes/test.py b/_examples/gobytes/test.py new file mode 100644 index 00000000..09887bc4 --- /dev/null +++ b/_examples/gobytes/test.py @@ -0,0 +1,18 @@ +# Copyright 2017 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from __future__ import print_function +import gobytes, go + +a = bytes([0, 1, 2, 3]) +b = gobytes.CreateBytes(10) +print ("Python bytes:", a) +print ("Go slice: ", b) + +print ("gobytes.HashBytes from Go bytes:", gobytes.HashBytes(b)) + +print("Python bytes to Go: ", go.Slice_byte.from_bytes(a)) +print("Go bytes to Python: ", bytes(go.Slice_byte([3, 4, 5]))) + +print("OK") diff --git a/_examples/slices/slices.go b/_examples/slices/slices.go index cccc8b3b..baa5d7e6 100644 --- a/_examples/slices/slices.go +++ b/_examples/slices/slices.go @@ -4,7 +4,10 @@ package slices -import "fmt" +import ( + "fmt" + "math/cmplx" +) func IntSum(s []int) int { sum := 0 @@ -28,6 +31,8 @@ type SliceInt16 []int16 type SliceInt32 []int32 type SliceInt64 []int64 +type SliceComplex []complex128 + type SliceIface []interface{} type S struct { @@ -47,3 +52,24 @@ func PrintSSlice(ss []*S) { func PrintS(s *S) { fmt.Printf("%v\n", s.Name) } + +func CmplxSqrt(arr SliceComplex) SliceComplex { + res := make([]complex128, len(arr)) + for i, el := range arr { + res[i] = cmplx.Sqrt(el) + } + return res +} + +func GetEmptyMatrix(xSize int, ySize int) [][]bool { + result := [][]bool{} + + for i := 0; i < xSize; i++ { + result = append(result, []bool{}) + for j := 0; j < ySize; j++ { + result[i] = append(result[i], false) + } + } + + return result +} diff --git a/_examples/slices/test.py b/_examples/slices/test.py index 1ed60b18..143f4533 100644 --- a/_examples/slices/test.py +++ b/_examples/slices/test.py @@ -3,6 +3,8 @@ # license that can be found in the LICENSE file. from __future__ import print_function +import math +import random import slices, go a = [1,2,3,4] @@ -35,4 +37,18 @@ slices.PrintS(ss[0]) slices.PrintS(ss[1]) +cmplx = slices.SliceComplex([(random.random() + random.random() * 1j) for _ in range(16)]) +sqrts = slices.CmplxSqrt(cmplx) +for root, orig in zip(sqrts, cmplx): + root_squared = root * root + assert math.isclose(root_squared.real, orig.real) + assert math.isclose(root_squared.imag, orig.imag) + + +matrix = slices.GetEmptyMatrix(4,4) +for i in range(4): + for j in range(4): + assert not matrix[i][j] +print("[][]bool working as expected") + print("OK") diff --git a/_examples/variadic/variadic.go b/_examples/variadic/variadic.go index 113ddf4a..9ed5dd72 100644 --- a/_examples/variadic/variadic.go +++ b/_examples/variadic/variadic.go @@ -4,7 +4,7 @@ package variadic -/////////////// Non Variadic ////////////// +// ///////////// Non Variadic ////////////// func NonVariFunc(arg1 int, arg2 []int, arg3 int) int { total := arg1 for _, num := range arg2 { @@ -15,7 +15,7 @@ func NonVariFunc(arg1 int, arg2 []int, arg3 int) int { return total } -/////////////// Variadic Over Int ////////////// +// ///////////// Variadic Over Int ////////////// func VariFunc(vargs ...int) int { total := 0 for _, num := range vargs { @@ -24,7 +24,7 @@ func VariFunc(vargs ...int) int { return total } -/////////////// Variadic Over Struct ////////////// +// ///////////// Variadic Over Struct ////////////// type IntStrUct struct { p int } @@ -43,7 +43,7 @@ func VariStructFunc(vargs ...IntStrUct) int { return total } -/////////////// Variadic Over Interface ////////////// +// ///////////// Variadic Over Interface ////////////// type IntInterFace interface { Number() int } diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index acded50d..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,42 +0,0 @@ -image: Previous Visual Studio 2019 - -build: off - -clone_folder: c:\gopath\src\github.com\go-python\gopy - -cache: - - '%LocalAppData%\\go-build' - - '%LocalAppData%\\pip' - -branches: - only: - - master - -environment: - GOPATH: C:\gopath - GOROOT: C:\go115 - GOPY_APPVEYOR_CI: '1' - GOTRACEBACK: 'crash' - #CPYTHON2DIR: "C:\\Python27-x64" - CPYTHON3DIR: "C:\\Python37-x64" - #PATH: '%GOPATH%\bin;%CPYTHON2DIR%;%CPYTHON2DIR%\\Scripts;%CPYTHON3DIR%;%CPYTHON3DIR%\\Scripts;C:\msys64\mingw64\bin;C:\msys64\usr\bin\;%PATH%' - PATH: '%GOPATH%\bin;%GOROOT%\bin;%CPYTHON3DIR%;%CPYTHON3DIR%\\Scripts;C:\msys64\mingw64\bin;C:\msys64\usr\bin\;%PATH%' - -stack: go 1.15 - -build_script: - - python --version - #- "%CPYTHON2DIR%\\python --version" - - "%CPYTHON3DIR%\\python --version" - #- "%CPYTHON2DIR%\\python -m pip install --upgrade pip" - - "%CPYTHON3DIR%\\python -m pip install --upgrade pip" - #- "%CPYTHON2DIR%\\python -m pip install cffi" - - "%CPYTHON3DIR%\\python -m pip install cffi" - #- "%CPYTHON2DIR%\\python -m pip install pybindgen" - - "%CPYTHON3DIR%\\python -m pip install pybindgen" - - go version - - go env - - go get -v -t ./... - -test_script: - - go test ./... diff --git a/bind/gen.go b/bind/gen.go index c36dc9be..fe96bd63 100644 --- a/bind/gen.go +++ b/bind/gen.go @@ -52,7 +52,9 @@ package main %[3]s // #define Py_LIMITED_API // need full API for PyRun* #include +#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L) typedef uint8_t bool; +#endif // static inline is trick for avoiding need for extra .c file // the following are used for build value -- switch on reflect.Kind // or the types equivalent @@ -85,10 +87,27 @@ static inline void gopy_err_handle() { PyErr_Print(); } } +// _gopy_clear_go_tls clears the Go goroutine pointer from this thread's TLS. +// When multiple gopy extensions share a process, each has its own Go runtime +// but all runtimes use the same TLS slot for the current goroutine pointer +// (GS:0x30 on darwin/amd64, FS:-8 on linux/amd64). After one extension's +// init(), TLS is left pointing to that runtime's g0. If another extension's +// CGo entry-point reads TLS and finds a non-nil goroutine, it takes the fast +// path (no needm()) and runs with the wrong M/P/mcache -- corrupting the heap. +// Clearing the slot before each CGo entry forces needm() to run, which +// establishes the correct per-extension context (issue #370). +static void _gopy_clear_go_tls(void) { +#if defined(__x86_64__) && defined(__APPLE__) + __asm__ volatile("movq $0, %%%%gs:0x30" ::: "memory"); +#elif defined(__x86_64__) && defined(__linux__) + __asm__ volatile("movq $0, %%%%fs:-8" ::: "memory"); +#endif +} %[8]s */ import "C" import ( + "runtime" "github.com/go-python/gopy/gopyh" // handler %[6]s ) @@ -130,6 +149,34 @@ func NumHandles() int { return gopyh.NumHandles() } +// _gcReq carries GC requests from RequestGC (called on a CGo/needm M) to a +// dedicated goroutine that actually calls runtime.GC(). Calling runtime.GC() +// directly from the gc.callbacks context (a CGo-needm M) races with goroutines +// mid-sweep on the same heap, causing "bad sweepgen in refill" panics on +// multi-core machines. Running GC on a proper goroutine eliminates that race. +// RequestGC blocks until the GC cycle completes, so Python memory measurements +// taken immediately after gc.collect() see the reclaimed Go memory. +var _gcReq = make(chan chan struct{}) + +func init() { + go func() { + for done := range _gcReq { + runtime.GC() + close(done) + } + }() +} + +// RequestGC runs Go's garbage collector synchronously and safely. +// gopy registers this via Python gc.callbacks so it fires after each Python +// GC cycle, keeping Go-heap objects freed via DecRef promptly collected. +//export RequestGC +func RequestGC() { + done := make(chan struct{}) + _gcReq <- done + <-done +} + // boolGoToPy converts a Go bool to python-compatible C.char func boolGoToPy(b bool) C.char { if b { @@ -147,7 +194,10 @@ func boolPyToGo(b C.char) bool { } func complex64GoToPy(c complex64) *C.PyObject { - return C.PyComplex_FromDoubles(C.double(real(c)), C.double(imag(c))) + gstate := C.PyGILState_Ensure() + obj := C.PyComplex_FromDoubles(C.double(real(c)), C.double(imag(c))) + C.PyGILState_Release(gstate) + return obj } func complex64PyToGo(o *C.PyObject) complex64 { @@ -156,7 +206,10 @@ func complex64PyToGo(o *C.PyObject) complex64 { } func complex128GoToPy(c complex128) *C.PyObject { - return C.PyComplex_FromDoubles(C.double(real(c)), C.double(imag(c))) + gstate := C.PyGILState_Ensure() + obj := C.PyComplex_FromDoubles(C.double(real(c)), C.double(imag(c))) + C.PyGILState_Release(gstate) + return obj } func complex128PyToGo(o *C.PyObject) complex128 { @@ -257,6 +310,8 @@ mod.add_function('GoPyInit', None, []) mod.add_function('DecRef', None, [param('int64_t', 'handle')]) mod.add_function('IncRef', None, [param('int64_t', 'handle')]) mod.add_function('NumHandles', retval('int'), []) +mod.add_function('RequestGC', None, []) +mod.add_function('_gopy_clear_go_tls', None, []) ` // appended to imports in py wrap preamble as key for adding at end @@ -279,8 +334,40 @@ except ImportError: cwd = os.getcwd() currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) os.chdir(currentdir) +# When multiple gopy extensions coexist in one Python process each carries its own +# independent Go runtime. Loading each extension without RTLD_GLOBAL below keeps its +# Go runtime symbols (including the per-runtime goroutine-pointer TLS slot) local to +# its own .so, which is what prevents the runtimes from colliding (issue #370). +# A _gopy_clear_go_tls() call before each CGo entry is available as an extra safety +# net but is opt-in (gopy build -clear-go-tls), off by default: on glibc + CPython +# 3.12+ its hardcoded TLS store clobbers the interpreter thread state and crashes on +# the first call in the common single-extension case (issue #395). +if hasattr(sys, 'getdlopenflags'): + try: + import ctypes as _gopy_ctypes + _gopy_saved_flags = sys.getdlopenflags() + sys.setdlopenflags(_gopy_saved_flags & ~getattr(_gopy_ctypes, 'RTLD_GLOBAL', 0)) + except Exception: + _gopy_saved_flags = None +else: + _gopy_saved_flags = None %[6]s +if _gopy_saved_flags is not None: + sys.setdlopenflags(_gopy_saved_flags) os.chdir(cwd) +# Run Go's GC whenever Python's GC runs so that Go-heap objects whose handles +# were released via DecRef are promptly collected. Without this, Go memory +# can accumulate between Python gc.collect() calls because Python GC only +# frees the Python wrapper; the underlying Go allocation is not reclaimed +# until Go's own GC fires. +try: + import gc as _gopy_gc + def _gopy_gc_cb(phase, info): + if phase == 'stop': + _%[1]s.RequestGC() + _gopy_gc.callbacks.append(_gopy_gc_cb) +except Exception: + pass # to use this code in your end-user python file, import it as follows: # from %[1]s import %[3]s @@ -409,8 +496,8 @@ build: # goimports is needed to ensure that the imports list is valid $(GOIMPORTS) -w %[1]s.go # this will otherwise be built during go build and may be out of date - - rm %[1]s.c - echo "typedef uint8_t bool;" > %[1]s_go.h + - rm %[1]s.c + printf "#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L)\ntypedef uint8_t bool;\n#endif\n" > %[1]s_go.h # this will fail but is needed to generate the .c file that then allows go build to work - $(PYTHON) build.py >/dev/null 2>&1 # generate %[1]s_go.h from %[1]s.go -- unfortunately no way to build .h only @@ -438,6 +525,17 @@ var NoWarn = false // NoMake turns off generation of Makefiles var NoMake = false +// ClearGoTLS controls whether generated wrappers emit a _gopy_clear_go_tls() +// call before every CGo entry point (issue #370). It is opt-in and off by +// default. The clear performs a hardcoded TLS store (movq $0, %gs:0x30 on +// darwin/amd64, movq $0, %fs:-8 on linux/amd64); with a single gopy extension +// in the process it is unnecessary, and on glibc + CPython 3.12+ that offset +// overlaps the TLS slot CPython uses for the current thread state, so the store +// nulls it and the interpreter segfaults on the first call (issue #395). +// Loading each extension without RTLD_GLOBAL, done unconditionally, already +// keeps each runtime's goroutine-pointer TLS local to its own .so. +var ClearGoTLS = false + // GenPyBind generates a .go file, build.py file to enable pybindgen to create python bindings, // and wrapper .py file(s) that are loaded as the interface to the package with shadow // python-side classes @@ -551,8 +649,12 @@ func (g *pyGen) genPkgWrapOut() { // note: must generate import string at end as imports can be added during processing impstr := "" for _, im := range g.pkg.pyimports { - if g.mode == ModeGen || g.mode == ModeBuild { - impstr += fmt.Sprintf("import %s\n", im) + if g.mode == ModeGen || g.mode == ModeBuild || g.mode == ModePkg { + if g.cfg.PkgPrefix != "" { + impstr += fmt.Sprintf("from %s import %s\n", g.cfg.PkgPrefix, im) + } else { + impstr += fmt.Sprintf("import %s\n", im) + } } else { impstr += fmt.Sprintf("from %s import %s\n", g.cfg.Name, im) } @@ -649,7 +751,7 @@ func (g *pyGen) genPyWrapPreamble() { impgenstr += fmt.Sprintf("import %s\n", "_"+g.cfg.Name) } impstr += fmt.Sprintf(GoPkgDefs, g.cfg.Name) - case g.mode == ModeGen || g.mode == ModeBuild: + case g.mode == ModeGen || g.mode == ModeBuild || g.mode == ModePkg: if g.cfg.PkgPrefix != "" { for _, name := range impgenNames { impgenstr += fmt.Sprintf("from %s import %s\n", g.cfg.PkgPrefix, name) @@ -859,3 +961,13 @@ func (g *pyGen) genGoPkg() { g.genType(sym, false, false) // not exttypes } } + +// genStringerCall generates a call to either self.String() or self.string() +// depending on RenameCase option +func (g *pyGen) genStringerCall() { + if g.cfg.RenameCase { + g.pywrap.Printf("return self.string()\n") + } else { + g.pywrap.Printf("return self.String()\n") + } +} diff --git a/bind/gen_func.go b/bind/gen_func.go index 8f2e606e..b2643343 100644 --- a/bind/gen_func.go +++ b/bind/gen_func.go @@ -261,10 +261,8 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) { } } - // release GIL g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n") if !rvIsErr && nres != 2 { - // reacquire GIL after return g.gofile.Printf("defer C.PyEval_RestoreThread(_saved_thread)\n") } @@ -338,6 +336,17 @@ if __err != nil { } } + // Clear the Go TLS goroutine slot before the CGo entry point so that + // Go's needm() runs and establishes the correct per-extension context. + // Without this, two extensions sharing the same process can corrupt + // each other's heap via TLS collision (issue #370). Opt-in and off by + // default: the hardcoded TLS store crashes CPython 3.12+ in the common + // single-extension case (issue #395), and RTLD_GLOBAL-local loading + // already isolates each runtime's goroutine-pointer TLS. + if ClearGoTLS { + g.pywrap.Printf("_%s._gopy_clear_go_tls()\n", pkgname) + } + // pywrap output mnm := fsym.ID() if isMethod { @@ -415,7 +424,6 @@ if __err != nil { if rvIsErr || nres == 2 { g.gofile.Printf("\n") - // reacquire GIL g.gofile.Printf("C.PyEval_RestoreThread(_saved_thread)\n") g.gofile.Printf("if __err != nil {\n") diff --git a/bind/gen_map.go b/bind/gen_map.go index 27c1d86d..06d3302d 100644 --- a/bind/gen_map.go +++ b/bind/gen_map.go @@ -134,7 +134,7 @@ otherwise parameter is a python list that we copy from if isStringer(m.obj) { g.pywrap.Printf("def __str__(self):\n") g.pywrap.Indent() - g.pywrap.Printf("return self.String()\n") + g.genStringerCall() g.pywrap.Outdent() g.pywrap.Printf("\n") } @@ -303,14 +303,25 @@ otherwise parameter is a python list that we copy from g.gofile.Outdent() g.gofile.Printf("}\n") if esym.go2py != "" { - g.gofile.Printf("return %s(v)%s\n", esym.go2py, esym.go2pyParenEx) + // If the go2py starts with handleFromPtr_, use &v, otherwise just v + val_str := "" + if strings.HasPrefix(esym.go2py, "handleFromPtr_") { + val_str = "&v" + } else { + val_str = "v" + } + g.gofile.Printf("return %s(%s)%s\n", esym.go2py, val_str, esym.go2pyParenEx) } else { g.gofile.Printf("return v\n") } g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'), [param('%s', 'handle'), param('%s', '_ky')])\n", slNm, esym.cpyname, PyHandle, ksym.cpyname) + if esym.cpyname == "char*" { + g.pybuild.Printf("add_checked_string_function(mod, '%s_elem', retval('%s'), [param('%s', 'handle'), param('%s', '_ky')])\n", slNm, esym.cpyname, PyHandle, ksym.cpyname) + } else { + g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'), [param('%s', 'handle'), param('%s', '_ky')])\n", slNm, esym.cpyname, PyHandle, ksym.cpyname) + } // contains g.gofile.Printf("//export %s_contains\n", slNm) diff --git a/bind/gen_slice.go b/bind/gen_slice.go index 46d6d179..1c16180c 100644 --- a/bind/gen_slice.go +++ b/bind/gen_slice.go @@ -129,7 +129,7 @@ otherwise parameter is a python list that we copy from if isStringer(m.obj) { g.pywrap.Printf("def __str__(self):\n") g.pywrap.Indent() - g.pywrap.Printf("return self.String()\n") + g.genStringerCall() g.pywrap.Outdent() } } @@ -246,7 +246,11 @@ otherwise parameter is a python list that we copy from g.pywrap.Indent() g.pywrap.Printf("if self.index < len(self):\n") g.pywrap.Indent() - g.pywrap.Printf("rv = _%s_elem(self.handle, self.index)\n", qNm) + if esym.hasHandle() { + g.pywrap.Printf("rv = %s(handle=_%s_elem(self.handle, self.index))\n", esym.pyPkgId(slc.gopkg), qNm) + } else { + g.pywrap.Printf("rv = _%s_elem(self.handle, self.index)\n", qNm) + } g.pywrap.Println("self.index = self.index + 1") g.pywrap.Println("return rv") g.pywrap.Outdent() @@ -273,6 +277,25 @@ otherwise parameter is a python list that we copy from g.pywrap.Outdent() g.pywrap.Outdent() } + + if slNm == "Slice_byte" { + g.pywrap.Printf("@staticmethod\n") + g.pywrap.Printf("def from_bytes(value):\n") + g.pywrap.Indent() + g.pywrap.Printf(`"""Create a Go []byte object from a Python bytes object""" +`) + g.pywrap.Printf("handle = _%s_from_bytes(value)\n", qNm) + g.pywrap.Printf("return Slice_byte(handle=handle)\n") + g.pywrap.Outdent() + g.pywrap.Printf("def __bytes__(self):\n") + g.pywrap.Indent() + g.pywrap.Printf(`"""Convert the slice to a bytes object.""" +`) + g.pywrap.Printf("return _%s_to_bytes(self.handle)\n", qNm) + g.pywrap.Outdent() + g.pywrap.Outdent() + + } } if !extTypes || !pyWrapOnly { @@ -302,18 +325,31 @@ otherwise parameter is a python list that we copy from g.gofile.Indent() g.gofile.Printf("s := deptrFromHandle_%s(handle)\n", slNm) if esym.go2py != "" { - if !esym.isPointer() && esym.isStruct() { - g.gofile.Printf("return %s(&(s[_idx]))%s\n", esym.go2py, esym.go2pyParenEx) + // If the go2py starts with handleFromPtr_, use reference &, otherwise just return the value + val_str := "" + if strings.HasPrefix(esym.go2py, "handleFromPtr_") { + val_str = "&(s[_idx])" } else { - g.gofile.Printf("return %s(s[_idx])%s\n", esym.go2py, esym.go2pyParenEx) + val_str = "s[_idx]" } + g.gofile.Printf("return %s(%s)%s\n", esym.go2py, val_str, esym.go2pyParenEx) } else { g.gofile.Printf("return s[_idx]\n") } g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, PyHandle) + var caller_owns_ret string + var transfer_ownership string + if esym.cpyname == "PyObject*" { + caller_owns_ret = ", caller_owns_return=True" + transfer_ownership = ", transfer_ownership=False" + } + if esym.cpyname == "char*" { + g.pybuild.Printf("add_checked_string_function(mod, '%s_elem', retval('%s'), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, PyHandle) + } else { + g.pybuild.Printf("mod.add_function('%s_elem', retval('%s'%s), [param('%s', 'handle'), param('int', 'idx')])\n", slNm, esym.cpyname, caller_owns_ret, PyHandle) + } if slc.isSlice() { g.gofile.Printf("//export %s_subslice\n", slNm) @@ -340,7 +376,7 @@ otherwise parameter is a python list that we copy from g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_set', None, [param('%s', 'handle'), param('int', 'idx'), param('%v', 'value')])\n", slNm, PyHandle, esym.cpyname) + g.pybuild.Printf("mod.add_function('%s_set', None, [param('%s', 'handle'), param('int', 'idx'), param('%v', 'value'%s)])\n", slNm, PyHandle, esym.cpyname, transfer_ownership) if slc.isSlice() { g.gofile.Printf("//export %s_append\n", slNm) @@ -355,7 +391,38 @@ otherwise parameter is a python list that we copy from g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s_append', None, [param('%s', 'handle'), param('%s', 'value')])\n", slNm, PyHandle, esym.cpyname) + g.pybuild.Printf("mod.add_function('%s_append', None, [param('%s', 'handle'), param('%s', 'value'%s)])\n", slNm, PyHandle, esym.cpyname, transfer_ownership) + } + + if slNm == "Slice_byte" { + g.gofile.Printf("//export Slice_byte_from_bytes\n") + g.gofile.Printf("func Slice_byte_from_bytes(o *C.PyObject) CGoHandle {\n") + g.gofile.Indent() + g.gofile.Printf("size := C.PyBytes_Size(o)\n") + g.gofile.Printf("ptr := unsafe.Pointer(C.PyBytes_AsString(o))\n") + g.gofile.Printf("data := make([]byte, size)\n") + g.gofile.Printf("tmp := unsafe.Slice((*byte)(ptr), size)\n") + g.gofile.Printf("copy(data, tmp)\n") + g.gofile.Printf("return handleFromPtr_Slice_byte(&data)\n") + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.gofile.Printf("//export Slice_byte_to_bytes\n") + g.gofile.Printf("func Slice_byte_to_bytes(handle CGoHandle) *C.PyObject {\n") + g.gofile.Indent() + g.gofile.Printf("s := deptrFromHandle_Slice_byte(handle)\n") + g.gofile.Printf("ptr := unsafe.Pointer(&s[0])\n") + g.gofile.Printf("size := len(s)\n") + if WindowsOS { + g.gofile.Printf("return C.PyBytes_FromStringAndSize((*C.char)(ptr), C.longlong(size))\n") + } else { + g.gofile.Printf("return C.PyBytes_FromStringAndSize((*C.char)(ptr), C.long(size))\n") + } + g.gofile.Outdent() + g.gofile.Printf("}\n\n") + + g.pybuild.Printf("mod.add_function('Slice_byte_from_bytes', retval('%s'%s), [param('PyObject*', 'o', transfer_ownership=False)])\n", PyHandle, caller_owns_ret) + g.pybuild.Printf("mod.add_function('Slice_byte_to_bytes', retval('PyObject*', caller_owns_return=True), [param('%s', 'handle')])\n", PyHandle) } } } diff --git a/bind/gen_struct.go b/bind/gen_struct.go index 815c0834..076d09aa 100644 --- a/bind/gen_struct.go +++ b/bind/gen_struct.go @@ -101,7 +101,7 @@ in which case a new Go object is constructed first } g.pywrap.Printf("def __str__(self):\n") g.pywrap.Indent() - g.pywrap.Printf("return self.String()\n") + g.genStringerCall() g.pywrap.Outdent() g.pywrap.Printf("\n") } @@ -227,7 +227,11 @@ func (g *pyGen) genStructMemberGetter(s *Struct, i int, f types.Object) { g.gofile.Outdent() g.gofile.Printf("}\n\n") - g.pybuild.Printf("mod.add_function('%s', retval('%s'), [param('%s', 'handle')])\n", cgoFn, ret.cpyname, PyHandle) + if ret.cpyname == "char*" { + g.pybuild.Printf("add_checked_string_function(mod, '%s', retval('%s'), [param('%s', 'handle')])\n", cgoFn, ret.cpyname, PyHandle) + } else { + g.pybuild.Printf("mod.add_function('%s', retval('%s'), [param('%s', 'handle')])\n", cgoFn, ret.cpyname, PyHandle) + } } func (g *pyGen) genStructMemberSetter(s *Struct, i int, f types.Object) { @@ -345,7 +349,7 @@ handle=A Go-side object is always initialized with an explicit handle=arg } g.pywrap.Printf("def __str__(self):\n") g.pywrap.Indent() - g.pywrap.Printf("return self.String()\n") + g.genStringerCall() g.pywrap.Outdent() g.pywrap.Printf("\n") } diff --git a/bind/gen_varconst.go b/bind/gen_varconst.go index e0906a52..f794af81 100644 --- a/bind/gen_varconst.go +++ b/bind/gen_varconst.go @@ -125,6 +125,16 @@ func (g *pyGen) genConstValue(c *Const) { val = "False" } g.pywrap.Printf("%s = %s\n", c.GoName(), val) + if c.doc != "" { + lns := strings.Split(c.doc, "\n") + g.pywrap.Printf(`"""`) + g.pywrap.Printf("\n") + for _, l := range lns { + g.pywrap.Printf("%s\n", l) + } + g.pywrap.Printf(`"""`) + g.pywrap.Printf("\n") + } } func (g *pyGen) genEnum(e *Enum) { diff --git a/bind/package.go b/bind/package.go index 8a87df57..cf346c6a 100644 --- a/bind/package.go +++ b/bind/package.go @@ -112,6 +112,7 @@ func (p *Package) getDoc(parent string, o types.Object) string { n := o.Name() switch tp := o.(type) { case *types.Const: + // Check for untyped consts for _, c := range p.doc.Consts { for _, cn := range c.Names { if n == cn { @@ -119,6 +120,26 @@ func (p *Package) getDoc(parent string, o types.Object) string { } } } + // Check for typed consts + scopeName := p.pkg.Scope().Lookup(n) + if scopeName == nil { + return "" + } + constType := scopeName.Type() + if constType == nil { + return "" + } + for _, t := range p.doc.Types { + if p.pkg.Path()+"."+t.Name == constType.String() { + for _, c := range t.Consts { + for _, cn := range c.Names { + if n == cn { + return c.Doc + } + } + } + } + } case *types.Var: if tp.IsField() && parent != "" { @@ -316,8 +337,15 @@ func (p *Package) process() error { funcs[name] = fv case *types.TypeName: - named := obj.Type().(*types.Named) - switch typ := named.Underlying().(type) { + typ := obj.Type() + if named, ok := typ.(*types.Named); ok { + typ = named.Underlying() + } else { + // we are dealing with a type alias to a type literal. + // this is a cursed feature used to do structural typing. + // just pass it as-is. + } + switch typ := typ.(type) { case *types.Struct: sv, err := newStruct(p, obj) if err != nil { diff --git a/bind/printer_test.go b/bind/printer_test.go index c533f39b..5c53567a 100644 --- a/bind/printer_test.go +++ b/bind/printer_test.go @@ -49,7 +49,7 @@ impl 1 impl 2 ` - str := string(out.Bytes()) + str := out.String() if !reflect.DeepEqual(str, want) { t.Fatalf("error:\nwant=%q\ngot =%q\n", want, str) } diff --git a/bind/symbols.go b/bind/symbols.go index a9249d26..67bb6a5b 100644 --- a/bind/symbols.go +++ b/bind/symbols.go @@ -385,7 +385,7 @@ type symtab struct { syms map[string]*symbol imports map[string]string // key is full path, value is unique name importNames map[string]string // package name to path map -- for detecting name conflicts - uniqName byte // char for making package name unique + uniqName int // index for making package name unique parent *symtab } @@ -435,13 +435,11 @@ func (sym *symtab) addImport(pkg *types.Package) string { } ep, exists = sym.importNames[nm] if exists && ep != p { - if sym.uniqName == 0 { - sym.uniqName = 'a' - } else { + if exists && ep != p { sym.uniqName++ + unm = fmt.Sprintf("s%d_%s", sym.uniqName, nm) + fmt.Printf("import conflict: existing: %s new: %s alias: %s\n", ep, p, unm) } - unm = string([]byte{sym.uniqName}) + nm - fmt.Printf("import conflict: existing: %s new: %s alias: %s\n", ep, p, unm) } sym.importNames[unm] = p sym.imports[p] = unm @@ -1085,24 +1083,32 @@ func (sym *symtab) addSignatureType(pkg *types.Package, obj types.Object, t type py2g := fmt.Sprintf("%s { ", nsig) + py2g += "_gstate := C.PyGILState_Ensure()\n" + // TODO: use strings.Builder if rets.Len() == 0 { - py2g += "if C.PyCallable_Check(_fun_arg) == 0 { return }\n" + py2g += "if C.PyCallable_Check(_fun_arg) == 0 {\n" + py2g += "C.PyGILState_Release(_gstate)\n" // Release GIL + py2g += "return\n" + py2g += "}\n" } else { zstr, err := sym.ZeroToGo(ret.Type(), rsym) if err != nil { return err } - py2g += fmt.Sprintf("if C.PyCallable_Check(_fun_arg) == 0 { return %s }\n", zstr) + py2g += "if C.PyCallable_Check(_fun_arg) == 0 {\n" + py2g += "C.PyGILState_Release(_gstate)\n" // Release GIL + py2g += fmt.Sprintf("return %s\n", zstr) + py2g += "}\n" } - py2g += "_gstate := C.PyGILState_Ensure()\n" + if nargs > 0 { bstr, err := sym.buildTuple(args, "_fcargs", "_fun_arg") if err != nil { return err } py2g += bstr + retstr - py2g += fmt.Sprintf("C.PyObject_CallObject(_fun_arg, _fcargs)\n") + py2g += "C.PyObject_CallObject(_fun_arg, _fcargs)\n" py2g += "C.gopy_decref(_fcargs)\n" } else { // TODO: methods not supported for no-args case -- requires self arg.. diff --git a/bind/utils.go b/bind/utils.go index 42ef5bcc..5c13c301 100644 --- a/bind/utils.go +++ b/bind/utils.go @@ -103,7 +103,13 @@ func (pc *PyConfig) AllFlags() string { // python VM (python, python2, python3, pypy, etc...) func GetPythonConfig(vm string) (PyConfig, error) { code := `import sys -import distutils.sysconfig as ds +try: + import sysconfig as ds + def _get_python_inc(): + return ds.get_path('include') +except ImportError: + import distutils.sysconfig as ds + _get_python_inc = ds.get_config_var import json import os version=sys.version_info.major @@ -133,7 +139,7 @@ else: print(json.dumps({ "version": sys.version_info.major, "minor": sys.version_info.minor, - "incdir": ds.get_python_inc(), + "incdir": _get_python_inc(), "libdir": ds.get_config_var("LIBDIR"), "libpy": ds.get_config_var("LIBRARY"), "shlibs": ds.get_config_var("SHLIBS"), @@ -180,11 +186,10 @@ else: raw.LibDir = filepath.ToSlash(raw.LibDir) // on windows these can be empty -- use include dir which is usu good + // replace suffix case insensitive 'include' with 'libs' if raw.LibDir == "" && raw.IncDir != "" { - raw.LibDir = raw.IncDir - if strings.HasSuffix(raw.LibDir, "include") { - raw.LibDir = raw.LibDir[:len(raw.LibDir)-len("include")] + "libs" - } + regexInc := regexp.MustCompile(`(?i)\binclude$`) + raw.LibDir = regexInc.ReplaceAllString(raw.IncDir, "libs") fmt.Printf("no LibDir -- copy from IncDir: %s\n", raw.LibDir) } @@ -193,21 +198,17 @@ else: fmt.Printf("no LibPy -- set to: %s\n", raw.LibPy) } - if strings.HasSuffix(raw.LibPy, ".a") { - raw.LibPy = raw.LibPy[:len(raw.LibPy)-len(".a")] - } - if strings.HasPrefix(raw.LibPy, "lib") { - raw.LibPy = raw.LibPy[len("lib"):] - } + raw.LibPy = strings.TrimSuffix(raw.LibPy, ".a") + raw.LibPy = strings.TrimPrefix(raw.LibPy, "lib") cfg.Version = raw.Version cfg.ExtSuffix = raw.ExtSuffix cfg.CFlags = strings.Join([]string{ - "-I" + raw.IncDir, + `"-I` + raw.IncDir + `"`, }, " ") cfg.LdFlags = strings.Join([]string{ - "-L" + raw.LibDir, - "-l" + raw.LibPy, + `"-L` + raw.LibDir + `"`, + `"-l` + raw.LibPy + `"`, raw.ShLibs, raw.SysLibs, }, " ") diff --git a/cmd_build.go b/cmd_build.go index 07226290..7e38997f 100644 --- a/cmd_build.go +++ b/cmd_build.go @@ -5,6 +5,7 @@ package main import ( + "bytes" "fmt" "log" "os" @@ -44,6 +45,7 @@ ex: cmd.Flag.Bool("symbols", true, "include symbols in output") cmd.Flag.Bool("no-warn", false, "suppress warning messages, which may be expected") cmd.Flag.Bool("no-make", false, "do not generate a Makefile, e.g., when called from Makefile") + cmd.Flag.Bool("clear-go-tls", false, "emit a _gopy_clear_go_tls() call before every CGo entry (issue #370); off by default, needed only when several gopy extensions share one process and known to crash CPython 3.12+ (issue #395)") cmd.Flag.Bool("dynamic-link", false, "whether to link output shared library dynamically to Python") cmd.Flag.String("build-tags", "", "build tags to be passed to `go build`") return cmd @@ -71,6 +73,7 @@ func gopyRunCmdBuild(cmdr *commander.Command, args []string) error { bind.NoWarn = cfg.NoWarn bind.NoMake = cfg.NoMake + bind.ClearGoTLS = cmdr.Flag.Lookup("clear-go-tls").Value.Get().(bool) for _, path := range args { bpkg, err := loadPackage(path, true, cfg.BuildTags) // build first @@ -124,7 +127,7 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { if mode == bind.ModeExe { of, err := os.Create(buildname + ".h") // overwrite existing - fmt.Fprintf(of, "typedef uint8_t bool;\n") + fmt.Fprintf(of, "#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 202311L)\ntypedef uint8_t bool;\n#endif\n") of.Close() fmt.Printf("%v build.py # will fail, but needed to generate .c file\n", cfg.VM) @@ -180,30 +183,67 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { // build the go shared library upfront to generate the header // needed by our generated cpython code - args := []string{"build", "-mod=mod", "-buildmode=c-shared"} + firstArgs := []string{"build", "-mod=mod", "-buildmode=c-shared"} if cfg.BuildTags != "" { - args = append(args, "-tags", cfg.BuildTags) + firstArgs = append(firstArgs, "-tags", cfg.BuildTags) } if !cfg.Symbols { // These flags will omit the various symbol tables, thereby // reducing the final size of the binary. From https://golang.org/cmd/link/ // -s Omit the symbol table and debug information // -w Omit the DWARF symbol table - args = append(args, "-ldflags=-s -w") + firstArgs = append(firstArgs, "-ldflags=-s -w") } - args = append(args, "-o", buildLib, ".") - fmt.Printf("go %v\n", strings.Join(args, " ")) - cmd = exec.Command("go", args...) + firstArgs = append(firstArgs, "-o", buildLib, ".") + fmt.Printf("go %v\n", strings.Join(firstArgs, " ")) + cmd = exec.Command("go", firstArgs...) cmdout, err = cmd.CombinedOutput() if err != nil { fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout)) return err } - // update the output name to the one with the ABI extension - args[len(args)-2] = modlib // we don't need this initial lib because we are going to relink os.Remove(buildLib) + // Build the final extension with symbol-visibility restriction so that + // Go runtime globals are not placed in the global dynamic-linker + // namespace. Two independently-loaded Go runtimes sharing those globals + // via RTLD_GLOBAL interposition corrupt each other's GC state (#370). + // This applies only to the second build, which is where PyInit__ + // exists and where the exported-symbols list is valid. + finalArgs := []string{"build", "-mod=mod", "-buildmode=c-shared"} + if cfg.BuildTags != "" { + finalArgs = append(finalArgs, "-tags", cfg.BuildTags) + } + var finalLdFlags []string + if !cfg.Symbols { + finalLdFlags = append(finalLdFlags, "-s", "-w") + } + switch runtime.GOOS { + case "darwin": + ef, ferr := os.CreateTemp("", "gopy-exports-*.txt") + if ferr == nil { + fmt.Fprintf(ef, "_PyInit__%s\n", cfg.Name) + ef.Close() + defer os.Remove(ef.Name()) + finalLdFlags = append(finalLdFlags, "-extldflags=-Wl,-exported_symbols_list,"+ef.Name()) + } + case "linux": + ef, ferr := os.CreateTemp("", "gopy-exports-*.map") + if ferr == nil { + fmt.Fprintf(ef, "{ global: PyInit__%s; local: *; };\n", cfg.Name) + ef.Close() + defer os.Remove(ef.Name()) + finalLdFlags = append(finalLdFlags, "-extldflags=-Wl,--version-script="+ef.Name()) + } + } + if len(finalLdFlags) > 0 { + finalArgs = append(finalArgs, "-ldflags="+strings.Join(finalLdFlags, " ")) + } + finalArgs = append(finalArgs, "-o", modlib, ".") + // args is still used below for the CGO env build; point it at finalArgs. + args := finalArgs + // generate c code fmt.Printf("%v build.py\n", cfg.VM) cmd = exec.Command(cfg.VM, "build.py") @@ -215,19 +255,28 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { if bind.WindowsOS { fmt.Printf("Doing windows sed hack to fix declspec for PyInit\n") - cmd = exec.Command("sed", "-i", "s/ PyInit_/ __declspec(dllexport) PyInit_/g", cfg.Name+".c") - cmdout, err = cmd.CombinedOutput() + fname := cfg.Name + ".c" + raw, err := os.ReadFile(fname) if err != nil { - fmt.Printf("cmd had error: %v output:\no%v\n", err, string(cmdout)) - return err + fmt.Printf("could not read %s: %+v", fname, err) + return fmt.Errorf("could not read %s: %w", fname, err) + } + raw = bytes.ReplaceAll(raw, []byte(" PyInit_"), []byte(" __declspec(dllexport) PyInit_")) + err = os.WriteFile(fname, raw, 0644) + if err != nil { + fmt.Printf("could not apply sed hack to fix declspec for PyInit: %+v", err) + return fmt.Errorf("could not apply sed hack to fix PyInit: %w", err) } } cflags := strings.Fields(strings.TrimSpace(pycfg.CFlags)) - cflags = append(cflags, "-fPIC", "-Ofast") + cflags = append(cflags, "-fPIC", "-O3", "-ffast-math") if include, exists := os.LookupEnv("GOPY_INCLUDE"); exists { cflags = append(cflags, "-I"+filepath.ToSlash(include)) } + if oldcflags, exists := os.LookupEnv("CGO_CFLAGS"); exists { + cflags = append(cflags, oldcflags) + } var ldflags []string if cfg.DynamicLinking { ldflags = strings.Fields(strings.TrimSpace(pycfg.LdDynamicFlags)) @@ -243,6 +292,9 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error { if libname, exists := os.LookupEnv("GOPY_PYLIB"); exists { ldflags = append(ldflags, "-l"+filepath.ToSlash(libname)) } + if oldldflags, exists := os.LookupEnv("CGO_LDFLAGS"); exists { + ldflags = append(ldflags, oldldflags) + } removeEmpty := func(src []string) []string { o := make([]string, 0, len(src)) diff --git a/cmd_exe.go b/cmd_exe.go index 60ee3ced..acbd51e4 100644 --- a/cmd_exe.go +++ b/cmd_exe.go @@ -58,6 +58,7 @@ ex: cmd.Flag.String("url", "https://github.com/go-python/gopy", "home page for project") cmd.Flag.Bool("no-warn", false, "suppress warning messages, which may be expected") cmd.Flag.Bool("no-make", false, "do not generate a Makefile, e.g., when called from Makefile") + cmd.Flag.Bool("clear-go-tls", false, "emit a _gopy_clear_go_tls() call before every CGo entry (issue #370); off by default, needed only when several gopy extensions share one process and known to crash CPython 3.12+ (issue #395)") cmd.Flag.Bool("dynamic-link", false, "whether to link output shared library dynamically to Python") cmd.Flag.String("build-tags", "", "build tags to be passed to `go build`") @@ -97,6 +98,7 @@ func gopyRunCmdExe(cmdr *commander.Command, args []string) error { bind.NoWarn = cfg.NoWarn bind.NoMake = cfg.NoMake + bind.ClearGoTLS = cmdr.Flag.Lookup("clear-go-tls").Value.Get().(bool) if cfg.Name == "" { path := args[0] diff --git a/cmd_gen.go b/cmd_gen.go index becd0f1b..8b767354 100644 --- a/cmd_gen.go +++ b/cmd_gen.go @@ -37,6 +37,7 @@ ex: cmd.Flag.Bool("rename", false, "rename Go symbols to python PEP snake_case") cmd.Flag.Bool("no-warn", false, "suppress warning messages, which may be expected") cmd.Flag.Bool("no-make", false, "do not generate a Makefile, e.g., when called from Makefile") + cmd.Flag.Bool("clear-go-tls", false, "emit a _gopy_clear_go_tls() call before every CGo entry (issue #370); off by default, needed only when several gopy extensions share one process and known to crash CPython 3.12+ (issue #395)") cmd.Flag.Bool("dynamic-link", false, "whether to link output shared library dynamically to Python") cmd.Flag.String("build-tags", "", "build tags to be passed to `go build`") return cmd @@ -69,6 +70,7 @@ func gopyRunCmdGen(cmdr *commander.Command, args []string) error { bind.NoWarn = cfg.NoWarn bind.NoMake = cfg.NoMake + bind.ClearGoTLS = cmdr.Flag.Lookup("clear-go-tls").Value.Get().(bool) for _, path := range args { bpkg, err := loadPackage(path, true, cfg.BuildTags) // build first diff --git a/cmd_pkg.go b/cmd_pkg.go index c0d6cba3..9891907c 100644 --- a/cmd_pkg.go +++ b/cmd_pkg.go @@ -55,6 +55,7 @@ ex: cmd.Flag.String("url", "https://github.com/go-python/gopy", "home page for project") cmd.Flag.Bool("no-warn", false, "suppress warning messages, which may be expected") cmd.Flag.Bool("no-make", false, "do not generate a Makefile, e.g., when called from Makefile") + cmd.Flag.Bool("clear-go-tls", false, "emit a _gopy_clear_go_tls() call before every CGo entry (issue #370); off by default, needed only when several gopy extensions share one process and known to crash CPython 3.12+ (issue #395)") cmd.Flag.Bool("dynamic-link", false, "whether to link output shared library dynamically to Python") cmd.Flag.String("build-tags", "", "build tags to be passed to `go build`") @@ -93,6 +94,7 @@ func gopyRunCmdPkg(cmdr *commander.Command, args []string) error { bind.NoWarn = cfg.NoWarn bind.NoMake = cfg.NoMake + bind.ClearGoTLS = cmdr.Flag.Lookup("clear-go-tls").Value.Get().(bool) if cfg.Name == "" { path := args[0] @@ -167,7 +169,7 @@ func buildPkgRecurse(odir, path, rootpath string, exmap map[string]struct{}, bui drs := Dirs(dir) for _, dr := range drs { _, ex := exmap[dr] - if ex || dr[0] == '.' || dr[0] == '_' { + if ex || dr[0] == '.' || dr[0] == '_' || dr == "internal" { continue } sp := filepath.Join(path, dr) diff --git a/dirs.go b/dirs.go index 4fb9a52b..7ee4c47d 100644 --- a/dirs.go +++ b/dirs.go @@ -5,12 +5,12 @@ package main import ( - "io/ioutil" + "os" ) // Dirs returns a slice of all the directories within a given directory func Dirs(path string) []string { - files, err := ioutil.ReadDir(path) + files, err := os.ReadDir(path) if err != nil { return nil } diff --git a/doc.go b/doc.go index 54dd8443..2bd13640 100644 --- a/doc.go +++ b/doc.go @@ -6,7 +6,7 @@ gopy generates (and compiles) language bindings that make it possible to call Go code and pass objects from Python. -Using gopy +# Using gopy gopy takes a Go package and generates bindings for all of the exported symbols. The exported symbols define the cross-language interface. @@ -14,13 +14,12 @@ symbols. The exported symbols define the cross-language interface. The gopy tool generates both an API stub in Python, and binding code in Go. Start with a Go package: - package hi + package hi - import "fmt" - - func Hello(name string) { - fmt.Println("Hello, %s!\n", name) - } + import "fmt" + func Hello(name string) { + fmt.Println("Hello, %s!\n", name) + } */ package main diff --git a/gen.go b/gen.go index 5eeb6ca2..549ee2ad 100644 --- a/gen.go +++ b/gen.go @@ -97,8 +97,8 @@ func loadPackage(path string, buildFirst bool, buildTags string) (*packages.Pack buildTagStr := fmt.Sprintf("\"%s\"", strings.Join(strings.Split(buildTags, ","), " ")) args = append(args, "-tags", buildTagStr) } - args = append(args, "-v", "path") - fmt.Printf("go %v\n", strings.Join(args, " ")) + args = append(args, "-v", path) + fmt.Printf("go %s\n", strings.Join(args, " ")) cmd := exec.Command("go", args...) cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout @@ -115,7 +115,8 @@ func loadPackage(path string, buildFirst bool, buildTags string) (*packages.Pack } // golang.org/x/tools/go/packages supports modules or GOPATH etc - bpkgs, err := packages.Load(&packages.Config{Mode: packages.LoadTypes}, path) + mode := packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedDeps | packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes + bpkgs, err := packages.Load(&packages.Config{Mode: mode}, path) if err != nil { log.Printf("error resolving import path [%s]: %v\n", path, diff --git a/go.mod b/go.mod index c88f8b28..b591801e 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,15 @@ module github.com/go-python/gopy -go 1.18 +go 1.22.0 require ( github.com/gonuts/commander v0.1.0 github.com/gonuts/flag v0.1.0 github.com/pkg/errors v0.9.1 - golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1 + golang.org/x/tools v0.29.0 ) require ( - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/sync v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 577b3fb2..4b74f8cf 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,13 @@ github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0 github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1 h1:Z3vE1sGlC7qiyFJkkDcZms8Y3+yV8+W7HmDSmuf71tM= -golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= diff --git a/gopyh/handle.go b/gopyh/handle.go index 8e39bbfc..469c864d 100644 --- a/gopyh/handle.go +++ b/gopyh/handle.go @@ -159,7 +159,7 @@ func DecRef(handle CGoHandle) { } } -// IncRef increments the reference count for the specified handle. +// IncRef increments the reference count for the specified handle. func IncRef(handle CGoHandle) { if handle < 1 { return diff --git a/main_test.go b/main_test.go index 12a01930..1127698c 100644 --- a/main_test.go +++ b/main_test.go @@ -7,7 +7,6 @@ package main import ( "bytes" "fmt" - "io/ioutil" "log" "os" "os/exec" @@ -23,32 +22,34 @@ import ( var ( testBackends = map[string]string{} features = map[string][]string{ - "_examples/hi": []string{"py3"}, // output is different for 2 vs. 3 -- only checking 3 output - "_examples/funcs": []string{"py2", "py3"}, - "_examples/sliceptr": []string{"py2", "py3"}, - "_examples/simple": []string{"py2", "py3"}, - "_examples/empty": []string{"py2", "py3"}, - "_examples/named": []string{"py2", "py3"}, - "_examples/structs": []string{"py2", "py3"}, - "_examples/consts": []string{"py2", "py3"}, // 2 doesn't report .666 decimals - "_examples/vars": []string{"py2", "py3"}, - "_examples/seqs": []string{"py2", "py3"}, - "_examples/cgo": []string{"py2", "py3"}, - "_examples/pyerrors": []string{"py2", "py3"}, - "_examples/iface": []string{"py3"}, // output order diff for 2, fails but actually works - "_examples/pointers": []string{"py2", "py3"}, - "_examples/arrays": []string{"py2", "py3"}, - "_examples/slices": []string{"py2", "py3"}, - "_examples/maps": []string{"py2", "py3"}, - "_examples/gostrings": []string{"py2", "py3"}, - "_examples/rename": []string{"py2", "py3"}, - "_examples/lot": []string{"py2", "py3"}, - "_examples/unicode": []string{"py3"}, // doesn't work for 2 - "_examples/osfile": []string{"py2", "py3"}, - "_examples/gopygc": []string{"py2", "py3"}, - "_examples/cstrings": []string{"py2", "py3"}, - "_examples/pkgconflict": []string{"py2", "py3"}, + "_examples/hi": []string{"py3"}, + "_examples/gobytes": []string{"py3"}, + "_examples/funcs": []string{"py3"}, + "_examples/sliceptr": []string{"py3"}, + "_examples/simple": []string{"py3"}, + "_examples/empty": []string{"py3"}, + "_examples/named": []string{"py3"}, + "_examples/structs": []string{"py3"}, + "_examples/consts": []string{"py3"}, + "_examples/vars": []string{"py3"}, + "_examples/seqs": []string{"py3"}, + "_examples/cgo": []string{"py3"}, + "_examples/pyerrors": []string{"py3"}, + "_examples/iface": []string{"py3"}, + "_examples/pointers": []string{"py3"}, + "_examples/arrays": []string{"py3"}, + "_examples/slices": []string{"py3"}, + "_examples/maps": []string{"py3"}, + "_examples/gostrings": []string{"py3"}, + "_examples/rename": []string{"py3"}, + "_examples/lot": []string{"py3"}, + "_examples/unicode": []string{"py3"}, + "_examples/osfile": []string{"py3"}, + "_examples/gopygc": []string{"py3"}, + "_examples/cstrings": []string{"py3"}, + "_examples/pkgconflict": []string{"py3"}, "_examples/variadic": []string{"py3"}, + "_examples/gilstring": []string{"py3"}, } testEnvironment = os.Environ() @@ -66,7 +67,7 @@ func TestGovet(t *testing.T) { cmd.Stderr = buf err := cmd.Run() if err != nil { - t.Fatalf("error running %s:\n%s\n%v", "go vet", string(buf.Bytes()), err) + t.Fatalf("error running %s:\n%s\n%v", "go vet", buf.String(), err) } } @@ -91,17 +92,17 @@ func TestGofmt(t *testing.T) { err = cmd.Run() if err != nil { - t.Fatalf("error running %s:\n%s\n%v", exe, string(buf.Bytes()), err) + t.Fatalf("error running %s:\n%s\n%v", exe, buf.String(), err) } if len(buf.Bytes()) != 0 { - t.Errorf("some files were not gofmt'ed:\n%s\n", string(buf.Bytes())) + t.Errorf("some files were not gofmt'ed:\n%s\n", buf.String()) } } func TestGoPyErrors(t *testing.T) { pyvm := testBackends["py3"] - workdir, err := ioutil.TempDir("", "gopy-") + workdir, err := os.MkdirTemp("", "gopy-") if err != nil { t.Fatalf("could not create workdir: %v\n", err) } @@ -130,7 +131,6 @@ ignoring python incompatible function: .func github.com/go-python/gopy/_examples func TestHi(t *testing.T) { // t.Parallel() path := "_examples/hi" - // NOTE: output differs for python2 -- only valid checking for 3 testPkg(t, pkg{ path: path, lang: features[path], @@ -216,7 +216,7 @@ caught: can't work for 24 hours! --- p.Salary(24): caught: can't work for 24 hours! --- Person.__init__ caught: argument 2 must be str, not int | err-type: -caught: an integer is required (got type str) | err-type: +caught: 'str' object cannot be interpreted as an integer | err-type: *ERROR* no exception raised! hi.Person{Name="name", Age=0} hi.Person{Name="name", Age=42} @@ -269,6 +269,25 @@ OK } +func TestBytes(t *testing.T) { + // t.Parallel() + path := "_examples/gobytes" + testPkg(t, pkg{ + path: path, + lang: features[path], + cmd: "build", + extras: nil, + want: []byte(`Python bytes: b'\x00\x01\x02\x03' +Go slice: go.Slice_byte len: 10 handle: 1 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +gobytes.HashBytes from Go bytes: gobytes.Array_4_byte len: 4 handle: 2 [12, 13, 81, 81] +Python bytes to Go: go.Slice_byte len: 4 handle: 3 [0, 1, 2, 3] +Go bytes to Python: b'\x03\x04\x05' +OK +`), + }) + +} + func TestBindFuncs(t *testing.T) { // t.Parallel() path := "_examples/funcs" @@ -594,10 +613,11 @@ slices.IntSum from Python list: 10 slices.IntSum from Go slice: 10 unsigned slice elements: 1 2 3 4 signed slice elements: -1 -2 -3 -4 -struct slice: slices.Slice_Ptr_slices_S len: 3 handle: 11 [12, 13, 14] +struct slice: slices.Slice_Ptr_slices_S len: 3 handle: 11 [slices.S{Name=S0, handle=12}, slices.S{Name=S1, handle=13}, slices.S{Name=S2, handle=14}] struct slice[0]: slices.S{Name=S0, handle=15} struct slice[1]: slices.S{Name=S1, handle=16} struct slice[2].Name: S2 +[][]bool working as expected OK `), }) @@ -763,6 +783,74 @@ OK }) } +// TestGilString is a regression test for the multi-runtime crash (issue #370). +// It replicates the exact reproduction from the issue report: two separately-built +// gopy extensions (gilstring and simple) loaded in the same Python process, with +// calls interleaved in a loop of 5000 iterations. +func TestGilString(t *testing.T) { + backends := []string{"py3"} + for _, be := range backends { + vm, ok := testBackends[be] + if !ok || vm == "" { + t.Logf("Skipped testing backend %s for TestGilString\n", be) + continue + } + t.Run(be, func(t *testing.T) { + cwd, _ := os.Getwd() + + workdir, err := os.MkdirTemp("", "gopy-") + if err != nil { + t.Fatalf("could not create workdir: %v", err) + } + defer os.RemoveAll(workdir) + defer bind.ResetPackages() + + gilDir := filepath.Join(workdir, "gilstring") + if err := os.MkdirAll(gilDir, 0700); err != nil { + t.Fatalf("could not create gilstring subdir: %v", err) + } + writeGoMod(t, cwd, gilDir) + if err := run([]string{"build", "-vm=" + vm, "-output=" + gilDir, "./_examples/gilstring"}); err != nil { + t.Fatalf("error building gilstring: %v", err) + } + bind.ResetPackages() + + simpleDir := filepath.Join(workdir, "simple") + if err := os.MkdirAll(simpleDir, 0700); err != nil { + t.Fatalf("could not create simple subdir: %v", err) + } + writeGoMod(t, cwd, simpleDir) + if err := run([]string{"build", "-vm=" + vm, "-output=" + simpleDir, "./_examples/simple"}); err != nil { + t.Fatalf("error building simple: %v", err) + } + + tstDst := filepath.Join(workdir, "test.py") + if err := copyCmd(filepath.Join(cwd, "_examples/gilstring/test.py"), tstDst); err != nil { + t.Fatalf("error copying test.py: %v", err) + } + + env := make([]string, len(testEnvironment)) + copy(env, testEnvironment) + env = append(env, fmt.Sprintf("PYTHONPATH=%s", workdir)) + + cmd := exec.Command(vm, "./test.py") + cmd.Env = env + cmd.Dir = workdir + cmd.Stdin = os.Stdin + buf, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("error running python module: err=%v\n%s", err, string(buf)) + } + + got := strings.Replace(string(buf), "\r\n", "\n", -1) + want := "OK\n" + if got != want { + t.Fatalf("got:\n%s\nwant:\n%s", got, want) + } + }) + } +} + func TestPackagePrefix(t *testing.T) { // t.Parallel() path := "_examples/package/mypkg" @@ -886,14 +974,14 @@ don't modify manually. } if os.Getenv("GOPY_GENERATE_SUPPORT_MATRIX") == "1" { - err := ioutil.WriteFile("SUPPORT_MATRIX.md", buf.Bytes(), 0644) + err := os.WriteFile("SUPPORT_MATRIX.md", buf.Bytes(), 0644) if err != nil { log.Fatalf("Unable to write SUPPORT_MATRIX.md") } return } - src, err := ioutil.ReadFile("SUPPORT_MATRIX.md") + src, err := os.ReadFile("SUPPORT_MATRIX.md") if err != nil { log.Fatalf("Unable to read SUPPORT_MATRIX.md") } @@ -922,7 +1010,6 @@ type pkg struct { } func testPkg(t *testing.T, table pkg) { - // backends := []string{"py2", "py3"} backends := []string{"py3"} // backends := table.lang // todo: enabling py2 testing requires separate "want" output for _, be := range backends { @@ -934,11 +1021,6 @@ func testPkg(t *testing.T, table pkg) { continue } switch be { - case "py2": - t.Run(be, func(t *testing.T) { - // t.Parallel() - testPkgBackend(t, vm, table) - }) case "py3": t.Run(be, func(t *testing.T) { // t.Parallel() @@ -958,7 +1040,7 @@ require github.com/go-python/gopy v0.0.0 replace github.com/go-python/gopy => %s ` contents := fmt.Sprintf(template, pkgDir) - if err := ioutil.WriteFile(filepath.Join(tstDir, "go.mod"), []byte(contents), 0666); err != nil { + if err := os.WriteFile(filepath.Join(tstDir, "go.mod"), []byte(contents), 0666); err != nil { t.Fatalf("failed to write go.mod file: %v", err) } } @@ -967,7 +1049,7 @@ func testPkgBackend(t *testing.T, pyvm string, table pkg) { curPkgPath := reflect.TypeOf(table).PkgPath() _, pkgNm := filepath.Split(table.path) cwd, _ := os.Getwd() - workdir, err := ioutil.TempDir("", "gopy-") + workdir, err := os.MkdirTemp("", "gopy-") if err != nil { t.Fatalf("[%s:%s]: could not create workdir: %v\n", pyvm, table.path, err) } @@ -1061,6 +1143,10 @@ func testPkgBackend(t *testing.T, pyvm string, table pkg) { diff, _ := cmd.CombinedOutput() diffTxt = string(diff) + "\n" } + t.Fatalf("[%s:%s]: error running python module:\n%s", + pyvm, table.path, + diffTxt, + ) } t.Fatalf("[%s:%s]: error running python module:\ngot:\n%s\n\nwant:\n%s\n[%s:%s] diff:\n%s", diff --git a/main_unix.go b/main_unix.go index 1acf66f9..bd84a696 100644 --- a/main_unix.go +++ b/main_unix.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux && !android) || dragonfly || openbsd -// +build linux,!android dragonfly openbsd +//go:build (linux && !android) || dragonfly || openbsd || freebsd +// +build linux,!android dragonfly openbsd freebsd package main diff --git a/main_unix_test.go b/main_unix_test.go index 0d709f74..f01f8e8c 100644 --- a/main_unix_test.go +++ b/main_unix_test.go @@ -19,9 +19,7 @@ func init() { testEnvironment = os.Environ() var ( - py2 = "python2" py3 = "python3" - // pypy2 = "pypy" // pypy3 = "pypy3" ) @@ -40,7 +38,6 @@ func init() { mandatory bool }{ {"py3", py3, "", true}, - {"py2", py2, "", true}, } { args := []string{"-c", ""} if be.module != "" { diff --git a/main_windows_test.go b/main_windows_test.go index 037b33ab..64d4aa9f 100644 --- a/main_windows_test.go +++ b/main_windows_test.go @@ -40,7 +40,6 @@ func init() { mandatory bool }{ {"py3", py3, "", true}, - // {"py2", py2, "", true}, } { args := []string{"-c", ""} if be.module != "" { diff --git a/pkgsetup.go b/pkgsetup.go index 40fb48e5..ded55bd2 100644 --- a/pkgsetup.go +++ b/pkgsetup.go @@ -19,6 +19,12 @@ const ( with open("README.md", "r") as fh: long_description = fh.read() + +class BinaryDistribution(setuptools.Distribution): + def has_ext_modules(_): + return True + + setuptools.setup( name="%[1]s%[2]s", version="%[3]s", @@ -35,6 +41,7 @@ setuptools.setup( "Operating System :: OS Independent", ], include_package_data=True, + distclass=BinaryDistribution, ) ` diff --git a/version.go b/version.go index 499dd88f..55a0f480 100644 --- a/version.go +++ b/version.go @@ -3,7 +3,7 @@ package main const ( - Version = "v0.4.5" - GitCommit = "cb06da2" // the commit JUST BEFORE the release - VersionDate = "2022-07-11 19:34" // UTC + Version = "v0.4.10" + GitCommit = "b735a58" // the commit JUST BEFORE the release + VersionDate = "2024-05-03 22:57" // UTC )