diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 30af44995..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,29 +0,0 @@ -[bumpversion] -current_version = 2.3.0 -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}.{release}{dev} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = dummy -values = - dev - dummy - -[bumpversion:part:dev] - -[bumpversion:file:setup.py] - -[bumpversion:file:conda.recipe/meta.yaml] - -[bumpversion:file:src/runtime/resources/clr.py] - -[bumpversion:file:src/SharedAssemblyInfo.cs] -serialize = - {major}.{minor}.{patch} - -[bumpversion:file:src/clrmodule/ClrModule.cs] -serialize = - {major}.{minor}.{patch} - diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..d2de0afdd --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,95 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/python:3.9.1 + + working_directory: ~/repo + + steps: + - checkout + + - restore_cache: + keys: + - v3-dependencies-{{ checksum "requirements.txt" }} + - v3-dependencies- + + - run: + name: install apt-transport-https + command: | + sudo apt-get update + sudo apt-get install apt-transport-https + + - run: + name: install dotnet + command: | + # see https://dotnet.microsoft.com/download/linux-package-manager/debian9/sdk-current + wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg + sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ + wget -q https://packages.microsoft.com/config/debian/9/prod.list + sudo mv prod.list /etc/apt/sources.list.d/microsoft-prod.list + sudo chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg + sudo chown root:root /etc/apt/sources.list.d/microsoft-prod.list + sudo apt-get update + sudo apt-get install dotnet-sdk-5.0 + sudo apt-get install aspnetcore-runtime-5.0 + sudo apt-get install dotnet-runtime-5.0 + + - run: + name: dotnet info + command: | + dotnet --info + + - run: + name: Install 7z, unrar + command: | + sudo apt-get install -y p7zip-full + + - run: + name: install dependencies + command: | + python3 -m venv venv + . venv/bin/activate + python3 -m pip install -r requirements.txt + + - save_cache: + paths: + - ./venv + key: v3-dependencies-{{ checksum "requirements.txt" }} + + - run: + name: build pythonnet + command: | + . venv/bin/activate + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + python3 setup.py build_dotnet + python3 setup.py bdist_wheel + python3 setup.py develop + + - run: + name: check installation + command: | + python3 -m venv venv + . venv/bin/activate + python -c "import clr" + + - run: + name: unittests + command: | + python3 -m venv venv + . venv/bin/activate + pytest + dotnet test src/embed_tests/ + + - run: + name: wheel + command: | + . venv/bin/activate + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + python3 setup.py bdist_wheel + mkdir -p test-reports/dist + cp dist/*.whl test-reports/dist + + - store_artifacts: + path: test-reports + destination: test-reports diff --git a/.editorconfig b/.editorconfig index 2e7c58ffe..7b07446e3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,10 +19,25 @@ indent_size = 2 [*.{csproj,pyproj,config}] indent_size = 2 +# .NET formatting settings +[*.{cs,vb}] +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = true + +[*.cs] +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true + # Solution [*.sln] indent_style = tab +[*.csproj] +charset = utf-8 +insert_final_newline = true + # bumpversion reformats itself after every bump [.bumpversion.cfg] trim_trailing_whitespace = false diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 2758080dc..c232a1186 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,6 +3,7 @@ - Pythonnet version: - Python version: - Operating System: +- .NET Runtime: ### Details diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..688d65c04 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,62 @@ +name: GitHub Actions + +on: [ pull_request, push ] + +jobs: + build-test: + name: Build and Test + runs-on: ${{ matrix.os }}-latest + timeout-minutes: 5 + + strategy: + fail-fast: false + matrix: + os: [windows, ubuntu, macos] + python: [3.6, 3.7, 3.8, 3.9] + platform: [x64] + shutdown_mode: [Normal, Soft] + + env: + PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }} + + steps: + - name: Set Environment on macOS + uses: maxim-lobanov/setup-xamarin@v1 + if: ${{ matrix.os == 'macos' }} + with: + mono-version: latest + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v1 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.platform }} + + - name: Install dependencies + run: | + pip install --upgrade -r requirements.txt + + - name: Build and Install + run: | + python setup.py configure + pip install -v . + + - name: Python Tests + run: pytest + + - name: Embedding tests + run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ + if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython + + - name: Python tests run from .NET + run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ + if: ${{ matrix.os == 'windows' }} # Not working for others right now + + # TODO: Run perf tests + # TODO: Run mono tests on Windows? diff --git a/.gitignore b/.gitignore index 6f813dcb0..673681317 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ +/src/runtime/interopNative.cs + +# Configuration data +configured.props + # General binaries and Build results *.dll *.exe *.pdb +*.deps.json ### JetBrains ### .idea/ @@ -15,6 +21,7 @@ __pycache__/ build/ dist/ *.egg-info/ +.eggs/ # Unit test / coverage reports htmlcov/ @@ -57,3 +64,10 @@ UpgradeLog*.htm # Coverity cov-int/ + +# Visual Studio Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json diff --git a/.mention-bot b/.mention-bot new file mode 100644 index 000000000..f07fa548e --- /dev/null +++ b/.mention-bot @@ -0,0 +1,35 @@ +{ + "maxReviewers": 5, + "numFilesToCheck": 10, + "message": "@pullRequester, thanks! @reviewers, please review this.", + "alwaysNotifyForPaths": [ + { + "name": "ghuser", + "files": ["src/js/**/*.js"], + "skipTeamPrs": false + } + ], + "fallbackNotifyForPaths": [ + { + "name": "ghuser", + "files": ["src/js/**/*.js"], + "skipTeamPrs": false + } + ], + "findPotentialReviewers": true, + "fileBlacklist": ["*.md"], + "userBlacklist": [], + "userBlacklistForPR": [], + "requiredOrgs": [], + "actions": ["opened"], + "skipAlreadyAssignedPR": false, + "skipAlreadyMentionedPR": false, + "assignToReviewer": false, + "createReviewRequest": false, + "createComment": true, + "skipTitle": "", + "withLabel": "", + "delayed": false, + "delayedUntil": "3d", + "skipCollaboratorPR": false +} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bf336dc9f..000000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ -sudo: false - -language: python -python: - - 2.7 - - 3.3 - - 3.4 - - 3.5 - - 3.6 - - 3.7-dev - -matrix: - allow_failures: - - python: 3.7-dev - -env: - global: - - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - - SEGFAULT_SIGNALS=all - - PYTHONUNBUFFERED=True - - CODECOV_ENV=TRAVIS_PYTHON_VERSION - -addons: - apt: - sources: - - mono - - mono-libtiff-compat - packages: - - mono-devel - - ca-certificates-mono - -before_install: - # Set-up dll path for embedded tests - - PY_LIBDIR=$(python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))') - - export LD_LIBRARY_PATH=$PY_LIBDIR:$LD_LIBRARY_PATH - -install: - - pip install --upgrade -r requirements.txt - - coverage run setup.py install - -script: - - python -m pytest - - mono ./packages/NUnit.*/tools/nunit3-console.exe src/embed_tests/bin/Python.EmbeddingTest.dll - -after_script: - # Uncomment if need to geninterop, ie. py37 final - # - python tools/geninterop/geninterop.py - - # Waiting on mono-coverage, SharpCover or xr.Baboon - - coverage xml -i - - codecov --file coverage.xml --flags setup_linux - -notifications: - email: false - slack: - secure: "UiQdSK1/uNnHl8/gQgfLj/F5JGxtJuaT3QYtKNcw3Ddpr3FX8tfXJ/RjsCsSlRQzDm7AdBAeMzcBQmvH4iRIV2y7qVywLyru5MPiwY4ZjMN6fJK/zaaxetOct9fasIBYzHguNPDAtiBGFh2iK1H1MXTY8rkmU3WZvl18b8EsrP0=" diff --git a/AUTHORS.md b/AUTHORS.md index 09358586e..167fd496c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -2,8 +2,11 @@ ## Development Lead -- Barton Cline ([@BartonCline](https://github.com/BartonCline)) - Benedikt Reinartz ([@filmor](https://github.com/filmor)) +- Victor Milovanov ([@lostmsu](https://github.com/lostmsu)) + +## Former Development Leads +- Barton Cline ([@BartonCline](https://github.com/BartonCline)) - Brian Lloyd ([@brianlloyd](https://github.com/brianlloyd)) - David Anthoff ([@davidanthoff](https://github.com/davidanthoff)) - Denis Akhiyarov ([@denfromufa](https://github.com/denfromufa)) @@ -12,31 +15,70 @@ ## Contributors +- Alex Earl ([@slide](https://github.com/slide)) +- Alex Helms ([@alexhelms](https://github.com/alexhelms)) +- Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) +- Andrey Sant'Anna ([@andreydani](https://github.com/andreydani)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) +- Avinash Maddikonda ([@SFM61319](https://github.com/SFM61319)) +- Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) +- Callum Noble ([@callumnoble](https://github.com/callumnoble)) +- Christabella Irwanto([@christabella](https://github.com/christabella)) - Christian Heimes ([@tiran](https://github.com/tiran)) - Christoph Gohlke ([@cgohlke](https://github.com/cgohlke)) +- Christopher Bremner ([@chrisjbremner](https://github.com/chrisjbremner)) +- Christopher Pow ([@christopherpow](https://github.com/christopherpow)) +- Daniel Abrahamsson ([@danabr](https://github.com/danabr)) - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) +- Dave Hirschfeld ([@dhirschfeld](https://github.com/dhirschfeld)) +- David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) +- Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) +- Inna Wiesel ([@inna-w](https://github.com/inna-w)) +- Ivan Cronyn ([@cronan](https://github.com/cronan)) +- Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) - Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Robbins ([@jeff17robbins](https://github.com/jeff17robbins)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) +- Joe Lidbetter ([@jmlidbetter](https://github.com/jmlidbetter)) +- Joe Savage ([@s4v4g3](https://github.com/s4v4g3)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) +- John Wilkes ([@jbw3](https://github.com/jbw3)) - Luke Stratman ([@lstratman](https://github.com/lstratman)) +- Konstantin Posudevskiy ([@konstantin-posudevskiy](https://github.com/konstantin-posudevskiy)) - Matthias Dittrich ([@matthid](https://github.com/matthid)) +- Meinrad Recheis ([@henon](https://github.com/henon)) +- Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) +- Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) +- Simon Mourier ([@smourier](https://github.com/smourier)) +- Tom Minka ([@tminka](https://github.com/tminka)) +- Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) +- Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) +- William Sardar ([@williamsardar])(https://github.com/williamsardar) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) +- ([@amos402]https://github.com/amos402) - ([@bltribble](https://github.com/bltribble)) +- ([@civilx64](https://github.com/civilx64)) +- ([@GSPP](https://github.com/GSPP)) - ([@omnicognate](https://github.com/omnicognate)) +- ([@OneBlue](https://github.com/OneBlue)) - ([@rico-chet](https://github.com/rico-chet)) - ([@rmadsen-ks](https://github.com/rmadsen-ks)) +- ([@SnGmng](https://github.com/SnGmng)) - ([@stonebig](https://github.com/stonebig)) +- ([@testrunner123](https://github.com/testrunner123)) +- ([@DanBarzilian](https://github.com/DanBarzilian)) +- ([@alxnull](https://github.com/alxnull)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c7de23c7..a9a804e8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,78 +1,231 @@ # Changelog -All notable changes to Python for .NET will be documented in this file. -This project adheres to [Semantic Versioning][]. +All notable changes to Python.NET will be documented in this file. This +project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. +## [Unreleased][] + +### Added + +- Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax +- Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). + +### Changed +- Drop support for Python 2, 3.4, and 3.5 +- `wchar_t` size aka `Runtime.UCS` is now determined at runtime +- `clr.AddReference` may now throw errors besides `FileNotFoundException`, that provide more +details about the cause of the failure +- `clr.AddReference` no longer adds ".dll" implicitly +- `PyIter(PyObject)` constructor replaced with static `PyIter.GetIter(PyObject)` method +- BREAKING: Return values from .NET methods that return an interface are now automatically + wrapped in that interface. This is a breaking change for users that rely on being + able to access members that are part of the implementation class, but not the + interface. Use the new __implementation__ or __raw_implementation__ properties to + if you need to "downcast" to the implementation class. +- BREAKING: Parameters marked with `ParameterAttributes.Out` are no longer returned in addition + to the regular method return value (unless they are passed with `ref` or `out` keyword). +- BREAKING: Drop support for the long-deprecated CLR.* prefix. +- `PyObject` now implements `IEnumerable` in addition to `IEnumerable` + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling +- Fixes issue with function resolution when calling overloaded function with keyword arguments from python ([#1097][i1097]) +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Fixed a bug where all .NET class instances were considered Iterable +- Fix incorrect choice of method to invoke when using keyword arguments. +- Fix non-delegate types incorrectly appearing as callable. +- Indexers can now be used with interface objects +- Fixed a bug where indexers could not be used if they were inherited +- Made it possible to use `__len__` also on `ICollection<>` interface objects +- Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions +- Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects +- Fixed objects returned by enumerating `PyObject` being disposed too soon +- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException +- `import` may now raise errors with more detail than "No module named X" + +### Removed + +- implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) + +## [2.5.0][] - 2020-06-14 + +This version improves performance on benchmarks significantly compared to 2.3. + +### Added + +- Automatic NuGet package generation in appveyor and local builds +- Function that sets `Py_NoSiteFlag` to 1. +- Support for Jetson Nano. +- Support for `__len__` for .NET classes that implement ICollection +- `PyExport` attribute to hide .NET types from Python +- `PythonException.Format` method to format exceptions the same as + `traceback.format_exception` +- `Runtime.None` to be able to pass `None` as parameter into Python from .NET +- `PyObject.IsNone()` to check if a Python object is None in .NET. +- Support for Python 3.8 +- Codecs as the designated way to handle automatic conversions between + .NET and Python types +- Added Python 3 buffer api support and PyBuffer interface for fast byte and numpy array read/write ([#980][p980]) + +### Changed + +- Added argument types information to "No method matches given arguments" message +- Moved wheel import in setup.py inside of a try/except to prevent pip collection failures +- Removes `PyLong_GetMax` and `PyClass_New` when targetting Python3 +- Improved performance of calls from Python to C# +- Added support for converting python iterators to C# arrays +- Changed usage of the obsolete function + `GetDelegateForFunctionPointer(IntPtr, Type)` to + `GetDelegateForFunctionPointer(IntPtr)` +- When calling C# from Python, enable passing argument of any type to a + parameter of C# type `object` by wrapping it into `PyObject` instance. + ([#881][i881]) +- Added support for kwarg parameters when calling .NET methods from Python +- Changed method for finding MSBuild using vswhere +- Reworked `Finalizer`. Now objects drop into its queue upon finalization, + which is periodically drained when new objects are created. +- Marked `Runtime.OperatingSystemName` and `Runtime.MachineName` as + `Obsolete`, should never have been `public` in the first place. They also + don't necessarily return a result that matches the `platform` module's. +- Unconditionally depend on `pycparser` for the interop module generation + +### Fixed + +- Fixed runtime that fails loading when using pythonnet in an environment + together with Nuitka +- Fixes bug where delegates get casts (dotnetcore) +- Determine size of interpreter longs at runtime +- Handling exceptions ocurred in ModuleObject's getattribute +- Fill `__classcell__` correctly for Python subclasses of .NET types +- Fixed issue with params methods that are not passed an array. +- Use UTF8 to encode strings passed to `PyRun_String` on Python 3 + +## [2.4.0][] - 2019-05-15 + +### Added + +- Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) +- Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). + Currently there two side-by-side build systems that produces the same output (net40) from the same sources. + After a some transition time, current (mono/ msbuild 14.0) build system will be removed. +- NUnit upgraded to 3.7 (eliminates travis-ci random bug) +- Added C# `PythonEngine.AddShutdownHandler` to help client code clean up on shutdown. +- Added `clr.GetClrType` ([#432][i432])([#433][p433]) +- Allowed passing `None` for nullable args ([#460][p460]) +- Added keyword arguments based on C# syntax for calling CPython methods ([#461][p461]) +- Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) +- Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) +- Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now ([#692][p692]). +- Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) +- Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added support for C# types to provide `__repr__` ([#680][p680]) + +### Changed + +- PythonException included C# call stack +- Reattach python exception traceback information (#545) +- PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) +- Refactored MethodBinder.Bind in preparation to make it extensible (#829) +- Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- Remove `LoadLibrary` call. ([#880][p880]) + +### Fixed + +- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. + This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. ([#534][p534]) +- Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py +- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), + related to unloading the Application Domain +- Fixed interop methods with Py_ssize_t. NetCoreApp 2.0 is more sensitive than net40 and requires this fix. ([#531][p531]) +- Fixed crash on exit of the Python interpreter if a python class + derived from a .NET class has a `__namespace__` or `__assembly__` + attribute ([#481][i481]) +- Fixed conversion of 'float' and 'double' values ([#486][i486]) +- Fixed 'clrmethod' for python 2 ([#492][i492]) +- Fixed double calling of constructor when deriving from .NET class ([#495][i495]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `LockRecursionException` when loading assemblies ([#627][i627]) +- Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) +- Fixed PyObject.GetHashCode ([#676][i676]) +- Fix memory leaks due to spurious handle incrementation ([#691][i691]) +- Fix spurious assembly loading exceptions from private types ([#703][i703]) +- Fix inheritance of non-abstract base methods ([#755][i755]) + + ## [2.3.0][] - 2017-03-11 ### Added -- Added Code Coverage (#345) -- Added `PySys_SetArgvEx` (#347) -- Added XML Documentation (#349) -- Added `Embedded_Tests` on AppVeyor (#224)(#353) -- Added `Embedded_Tests` on Travis (#224)(#391) -- Added PY3 settings to solution configuration-manager (#346) -- Added `Slack` (#384)(#383)(#386) +- Added Code Coverage ([#345][p345]) +- Added `PySys_SetArgvEx` ([#347][p347]) +- Added XML Documentation ([#349][p349]) +- Added `Embedded_Tests` on AppVeyor ([#224][i224])([#353][p353]) +- Added `Embedded_Tests` on Travis ([#224][i224])([#391][p391]) +- Added PY3 settings to solution configuration-manager ([#346][p346]) +- Added `Slack` ([#384][p384])([#383][i383])([#386][p386]) - Added function of passing an arbitrary .NET object as the value - of an attribute of `PyObject` (#370)(#373) -- Added `Coverity scan` (#390) -- Added `bumpversion` for version control (#319)(#398) -- Added `tox` for local testing (#345) + of an attribute of `PyObject` ([#370][i370])([#373][p373]) +- Added `Coverity scan` ([#390][i390]) +- Added `bumpversion` for version control ([#319][i319])([#398][p398]) +- Added `tox` for local testing ([#345][p345]) - Added `requirements.txt` -- Added to `PythonEngine` methods `Eval` and `Exec` (#389) -- Added implementations of `ICustomMarshal` (#407) -- Added docker images (#322) -- Added hooks in `pyinstaller` and `cx_freeze` for `pythonnet` (#66) +- Added to `PythonEngine` methods `Eval` and `Exec` ([#389][p389]) +- Added implementations of `ICustomMarshal` ([#407][p407]) +- Added docker images ([#322][i322]) +- Added hooks in `pyinstaller` and `cx_freeze` for `pythonnet` ([#66][i66]) ### Changed -- Refactored python `unittests` (#329) -- Refactored python `setup.py` (#337) -- Refactored remaining of Build Directives on `runtime.cs` (#339) -- Refactored `Embedded_Tests` to make easier to write tests (#369) -- Changed `unittests` to `pytest` (#368) -- Upgraded NUnit framework from `2.6.3` to `3.5.0` (#341) -- Downgraded NUnit framework from `3.5.0` to `2.6.4` (#353) -- Upgraded NUnit framework from `2.6.4` to `3.6.0` (#371) -- Unfroze Mono version on Travis (#345) -- Changed `conda.recipe` build to only pull-requests (#345) -- Combine `Py_DEBUG` and `PYTHON_WITH_PYDEBUG` flags (#362) +- Refactored python `unittests` ([#329][p329]) +- Refactored python `setup.py` ([#337][p337]) +- Refactored remaining of Build Directives on `runtime.cs` ([#339][p339]) +- Refactored `Embedded_Tests` to make easier to write tests ([#369][p369]) +- Changed `unittests` to `pytest` ([#368][p368]) +- Upgraded NUnit framework from `2.6.3` to `3.5.0` ([#341][p341]) +- Downgraded NUnit framework from `3.5.0` to `2.6.4` ([#353][p353]) +- Upgraded NUnit framework from `2.6.4` to `3.6.0` ([#371][p371]) +- Unfroze Mono version on Travis ([#345][p345]) +- Changed `conda.recipe` build to only pull-requests ([#345][p345]) +- Combine `Py_DEBUG` and `PYTHON_WITH_PYDEBUG` flags ([#362][i362]) ### Deprecated -- Deprecated `RunString` (#401) +- Deprecated `RunString` ([#401][i401]) ### Fixed -- Fixed crash during Initialization (#262)(#343) -- Fixed crash during Shutdown (#365) +- Fixed crash during Initialization ([#262][i262])([#343][p343]) +- Fixed crash during Shutdown ([#365][p365]) - Fixed multiple build warnings -- Fixed method signature match for Object Type (#203)(#377) -- Fixed outdated version number in AssemblyInfo (#398) -- Fixed wrong version number in `conda.recipe` (#398) +- Fixed method signature match for Object Type ([#203][i203])([#377][p377]) +- Fixed outdated version number in AssemblyInfo ([#398][p398]) +- Fixed wrong version number in `conda.recipe` ([#398][p398]) - Fixed fixture location for Python tests and `Embedded_Tests` -- Fixed `PythonException` crash during Shutdown (#400) -- Fixed `AppDomain` unload during GC (#397)(#400) -- Fixed `Py_Main` & `PySys_SetArgvEx` `no mem error` on `UCS4/PY3` (#399) -- Fixed `Python.Runtime.dll.config` on macOS (#120) -- Fixed crash on `PythonEngine.Version` (#413) -- Fixed `PythonEngine.PythonPath` issues (#179)(#414)(#415) +- Fixed `PythonException` crash during Shutdown ([#400][p400]) +- Fixed `AppDomain` unload during GC ([#397][i397])([#400][p400]) +- Fixed `Py_Main` & `PySys_SetArgvEx` `no mem error` on `UCS4/PY3` ([#399][p399]) +- Fixed `Python.Runtime.dll.config` on macOS ([#120][i120]) +- Fixed crash on `PythonEngine.Version` ([#413][i413]) +- Fixed `PythonEngine.PythonPath` issues ([#179][i179])([#414][i414])([#415][p415]) +- Fixed missing information on 'No method matches given arguments' by adding the method name ### Removed -- Removed `six` dependency for `unittests` (#329) -- Removed `Mono.Unix` dependency for `UCS4` (#360) +- Removed `six` dependency for `unittests` ([#329][p329]) +- Removed `Mono.Unix` dependency for `UCS4` ([#360][p360]) - Removed need for `Python.Runtime.dll.config` -- Removed PY32 build option `PYTHON_WITH_WIDE_UNICODE` (#417) +- Removed PY32 build option `PYTHON_WITH_WIDE_UNICODE` ([#417][i417]) ## [2.2.2][] - 2017-01-29 ### Fixed -- Missing files from packaging (#336) +- Missing files from packaging ([#336][i336]) ## [2.2.1][] - 2017-01-26 @@ -80,65 +233,65 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added -- Python 3.6 support (#310) -- Added `__version__` to module (#312) -- Added `conda` recipe (#281) -- Nuget update on build (#268) -- Added `__cause__` attribute on exception (#287) +- Python 3.6 support ([#310][p310]) +- Added `__version__` to module ([#312][p312]) +- Added `conda` recipe ([#281][p281]) +- Nuget update on build ([#268][p268]) +- Added `__cause__` attribute on exception ([#287][p287]) ### Changed -- License to MIT (#314) -- Project clean-up (#320) +- License to MIT ([#314][p314]) +- Project clean-up ([#320][p320]) - Refactor `#if` directives -- Rename Decref/Incref to XDecref/XIncre (#275) -- Remove printing if Decref is called with NULL (#275) +- Rename Decref/Incref to XDecref/XIncre ([#275][p275]) +- Remove printing if Decref is called with NULL ([#275][p275]) ### Removed -- Python 2.6 support (#270) -- Python 3.2 support (#270) +- Python 2.6 support ([#270][i270]) +- Python 3.2 support ([#270][i270]) ### Fixed -- Fixed `isinstance` refcount_leak (#273) -- Comparison Operators (#294) -- Improved Linux support (#300) -- Exception pickling (#286) +- Fixed `isinstance` refcount_leak ([#273][p273]) +- Comparison Operators ([#294][p294]) +- Improved Linux support ([#300][p300]) +- Exception pickling ([#286][p286]) ## [2.2.0-dev1][] - 2016-09-19 ### Changed -- Switch to C# 6.0 (#219) -- `setup.py` improvements for locating build tools (#208) -- unmanaged exports updated (#206) -- Mono update pinned to 4.2.4.4 (#233) +- Switch to C# 6.0 ([#219][p219]) +- `setup.py` improvements for locating build tools ([#208][p208]) +- unmanaged exports updated ([#206][p206]) +- Mono update pinned to 4.2.4.4 ([#233][p233]) ### Fixed -- Fixed relative imports (#219) -- Fixed recursive types (#250) -- Demo fix - stream reading (#225) +- Fixed relative imports ([#219][p219]) +- Fixed recursive types ([#250][p250]) +- Demo fix - stream reading ([#225][p225]) ## [2.1.0][] - 2016-04-12 ### Added -- Added Python 3.2 support. (#78) -- Added Python 3.3 support. (#78) -- Added Python 3.4 support. (#78) -- Added Python 3.5 support. (#163) -- Managed types can be sub-classed in Python (#78) -- Uses dynamic objects for cleaner code when embedding Python (#78) +- Added Python 3.2 support. ([#78][p78]) +- Added Python 3.3 support. ([#78][p78]) +- Added Python 3.4 support. ([#78][p78]) +- Added Python 3.5 support. ([#163][p163]) +- Managed types can be sub-classed in Python ([#78][p78]) +- Uses dynamic objects for cleaner code when embedding Python ([#78][p78]) ### Changed -- Better Linux support (with or without --enable-shared option) (#78) +- Better Linux support (with or without --enable-shared option) ([#78][p78]) ### Removed -- Implicit Type Casting (#131) +- Implicit Type Casting ([#131][i131]) ## [2.0.0][] - 2015-06-26 @@ -558,3 +711,101 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [2.0.0]: ../../compare/1.0...v2.0.0 [1.0.0]: https://github.com/pythonnet/pythonnet/releases/tag/1.0 + +[i714]: https://github.com/pythonnet/pythonnet/issues/714 +[i608]: https://github.com/pythonnet/pythonnet/issues/608 +[i443]: https://github.com/pythonnet/pythonnet/issues/443 +[p690]: https://github.com/pythonnet/pythonnet/pull/690 +[i475]: https://github.com/pythonnet/pythonnet/issues/475 +[p693]: https://github.com/pythonnet/pythonnet/pull/693 +[i432]: https://github.com/pythonnet/pythonnet/issues/432 +[p433]: https://github.com/pythonnet/pythonnet/pull/433 +[p460]: https://github.com/pythonnet/pythonnet/pull/460 +[p461]: https://github.com/pythonnet/pythonnet/pull/461 +[p433]: https://github.com/pythonnet/pythonnet/pull/433 +[i434]: https://github.com/pythonnet/pythonnet/issues/434 +[i481]: https://github.com/pythonnet/pythonnet/issues/481 +[i486]: https://github.com/pythonnet/pythonnet/issues/486 +[i492]: https://github.com/pythonnet/pythonnet/issues/492 +[i495]: https://github.com/pythonnet/pythonnet/issues/495 +[p607]: https://github.com/pythonnet/pythonnet/pull/607 +[i627]: https://github.com/pythonnet/pythonnet/issues/627 +[i276]: https://github.com/pythonnet/pythonnet/issues/276 +[i676]: https://github.com/pythonnet/pythonnet/issues/676 +[p345]: https://github.com/pythonnet/pythonnet/pull/345 +[p347]: https://github.com/pythonnet/pythonnet/pull/347 +[p349]: https://github.com/pythonnet/pythonnet/pull/349 +[i224]: https://github.com/pythonnet/pythonnet/issues/224 +[p353]: https://github.com/pythonnet/pythonnet/pull/353 +[p391]: https://github.com/pythonnet/pythonnet/pull/391 +[p346]: https://github.com/pythonnet/pythonnet/pull/346 +[p384]: https://github.com/pythonnet/pythonnet/pull/384 +[i383]: https://github.com/pythonnet/pythonnet/issues/383 +[p386]: https://github.com/pythonnet/pythonnet/pull/386 +[i370]: https://github.com/pythonnet/pythonnet/issues/370 +[p373]: https://github.com/pythonnet/pythonnet/pull/373 +[i390]: https://github.com/pythonnet/pythonnet/issues/390 +[i319]: https://github.com/pythonnet/pythonnet/issues/319 +[p398]: https://github.com/pythonnet/pythonnet/pull/398 +[p345]: https://github.com/pythonnet/pythonnet/pull/345 +[p389]: https://github.com/pythonnet/pythonnet/pull/389 +[p407]: https://github.com/pythonnet/pythonnet/pull/407 +[i322]: https://github.com/pythonnet/pythonnet/issues/322 +[i66]: https://github.com/pythonnet/pythonnet/issues/66 +[p329]: https://github.com/pythonnet/pythonnet/pull/329 +[p337]: https://github.com/pythonnet/pythonnet/pull/337 +[p339]: https://github.com/pythonnet/pythonnet/pull/339 +[p369]: https://github.com/pythonnet/pythonnet/pull/369 +[p368]: https://github.com/pythonnet/pythonnet/pull/368 +[p341]: https://github.com/pythonnet/pythonnet/pull/341 +[p353]: https://github.com/pythonnet/pythonnet/pull/353 +[p371]: https://github.com/pythonnet/pythonnet/pull/371 +[p345]: https://github.com/pythonnet/pythonnet/pull/345 +[i362]: https://github.com/pythonnet/pythonnet/issues/362 +[i401]: https://github.com/pythonnet/pythonnet/issues/401 +[i262]: https://github.com/pythonnet/pythonnet/issues/262 +[p343]: https://github.com/pythonnet/pythonnet/pull/343 +[p365]: https://github.com/pythonnet/pythonnet/pull/365 +[i203]: https://github.com/pythonnet/pythonnet/issues/203 +[p377]: https://github.com/pythonnet/pythonnet/pull/377 +[p398]: https://github.com/pythonnet/pythonnet/pull/398 +[p400]: https://github.com/pythonnet/pythonnet/pull/400 +[i397]: https://github.com/pythonnet/pythonnet/issues/397 +[p399]: https://github.com/pythonnet/pythonnet/pull/399 +[i120]: https://github.com/pythonnet/pythonnet/issues/120 +[i413]: https://github.com/pythonnet/pythonnet/issues/413 +[i179]: https://github.com/pythonnet/pythonnet/issues/179 +[i414]: https://github.com/pythonnet/pythonnet/issues/414 +[p415]: https://github.com/pythonnet/pythonnet/pull/415 +[p329]: https://github.com/pythonnet/pythonnet/pull/329 +[p360]: https://github.com/pythonnet/pythonnet/pull/360 +[i417]: https://github.com/pythonnet/pythonnet/issues/417 +[i336]: https://github.com/pythonnet/pythonnet/issues/336 +[p310]: https://github.com/pythonnet/pythonnet/pull/310 +[p312]: https://github.com/pythonnet/pythonnet/pull/312 +[p281]: https://github.com/pythonnet/pythonnet/pull/281 +[p268]: https://github.com/pythonnet/pythonnet/pull/268 +[p287]: https://github.com/pythonnet/pythonnet/pull/287 +[p314]: https://github.com/pythonnet/pythonnet/pull/314 +[p320]: https://github.com/pythonnet/pythonnet/pull/320 +[p275]: https://github.com/pythonnet/pythonnet/pull/275 +[i270]: https://github.com/pythonnet/pythonnet/issues/270 +[p273]: https://github.com/pythonnet/pythonnet/pull/273 +[p294]: https://github.com/pythonnet/pythonnet/pull/294 +[p300]: https://github.com/pythonnet/pythonnet/pull/300 +[p286]: https://github.com/pythonnet/pythonnet/pull/286 +[p219]: https://github.com/pythonnet/pythonnet/pull/219 +[p208]: https://github.com/pythonnet/pythonnet/pull/208 +[p206]: https://github.com/pythonnet/pythonnet/pull/206 +[p233]: https://github.com/pythonnet/pythonnet/pull/233 +[p219]: https://github.com/pythonnet/pythonnet/pull/219 +[p250]: https://github.com/pythonnet/pythonnet/pull/250 +[p225]: https://github.com/pythonnet/pythonnet/pull/225 +[p78]: https://github.com/pythonnet/pythonnet/pull/78 +[p163]: https://github.com/pythonnet/pythonnet/pull/163 +[p625]: https://github.com/pythonnet/pythonnet/pull/625 +[i131]: https://github.com/pythonnet/pythonnet/issues/131 +[p531]: https://github.com/pythonnet/pythonnet/pull/531 +[i755]: https://github.com/pythonnet/pythonnet/pull/755 +[p534]: https://github.com/pythonnet/pythonnet/pull/534 +[i449]: https://github.com/pythonnet/pythonnet/issues/449 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea0f3408e..ffeb792f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,15 @@ # How to contribute -PythonNet is developed and maintained by unpaid community members so well +Python.NET is developed and maintained by unpaid community members so well written, documented and tested pull requests are encouraged. By submitting a pull request for this project, you agree to license your contribution under the MIT license to this project. +This project has adopted the code of conduct defined by the Contributor +Covenant to clarify expected behavior in our community. For more information +see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). + ## Getting Started - Make sure you have a [GitHub account](https://github.com/signup/free) @@ -41,3 +45,4 @@ contribution under the MIT license to this project. - [General GitHub documentation](https://help.github.com/) - [GitHub pull request documentation](https://help.github.com/send-pull-requests/) +- [.NET Foundation Code of Conduct](https://dotnetfoundation.org/about/code-of-conduct) diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 000000000..5ad0c0e77 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,17 @@ + + + 3.0.0 + Copyright (c) 2006-2020 The Contributors of the Python.NET Project + pythonnet + Python.NET + 7.3 + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/LICENSE b/LICENSE index e344a0795..19c31a12f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2006-2017 the contributors of the "Python for .NET" project +Copyright (c) 2006-2020 the contributors of the Python.NET project Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), diff --git a/README.md b/README.md deleted file mode 100644 index 2794298a6..000000000 --- a/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# pythonnet - Python for .NET - -[![appveyor shield][]](https://ci.appveyor.com/project/pythonnet/pythonnet/branch/master) -[![travis shield][]](https://travis-ci.org/pythonnet/pythonnet) -[![codecov shield][]](https://codecov.io/github/pythonnet/pythonnet) -[![coverity shield][]](https://scan.coverity.com/projects/pythonnet) - -[![license shield][]](./LICENSE) -[![pypi package version][]](https://pypi.python.org/pypi/pythonnet) -[![python supported shield][]](https://pypi.python.org/pypi/pythonnet) -[![stackexchange shield][]](http://stackoverflow.com/questions/tagged/python.net) -[![slack][]](https://pythonnet.slack.com) - -Python for .NET is a package that gives Python programmers nearly -seamless integration with the .NET Common Language Runtime (CLR) and -provides a powerful application scripting tool for .NET developers. -It allows Python code to interact with the CLR, and may also be used to -embed Python into a .NET application. - -## Calling .NET code from Python - -Python for .NET allows CLR namespaces to be treated essentially -as Python packages. - -```python -import clr -from System import String -from System.Collections import * -``` - -To load an assembly, use the `AddReference` function in the `clr` module: - -```python -import clr -clr.AddReference("System.Windows.Forms") -from System.Windows.Forms import Form -``` - -## Embedding Python in .NET - -- All calls to python should be inside - a `using (Py.GIL()) {/* Your code here */}` block. -- Import python modules using `dynamic mod = Py.Import("mod")`, - then you can call functions as normal, eg `mod.func(args)`. -- Use `mod.func(args, Py.kw("keywordargname", keywordargvalue))` - to apply keyword arguments. -- All python objects should be declared as `dynamic` type. -- Mathematical operations involving python and literal/managed types must - have the python object first, eg. `np.pi * 2` works, `2 * np.pi` doesn't. - -### Example - -```csharp -static void Main(string[] args) -{ - using (Py.GIL()) - { - dynamic np = Py.Import("numpy"); - Console.WriteLine(np.cos(np.pi * 2)); - - dynamic sin = np.sin; - Console.WriteLine(sin(5)); - - double c = np.cos(5) + sin(5); - Console.WriteLine(c); - /* this block is temporarily disabled due to regression #249 - dynamic a = np.array(new List { 1, 2, 3 }); - Console.WriteLine(a.dtype); - - dynamic b = np.array(new List { 6, 5, 4 }, Py.kw("dtype", np.int32)); - Console.WriteLine(b.dtype); - - Console.WriteLine(a * b); - */ - Console.ReadKey(); - } -} -``` - -Output: - -```c -1.0 --0.958924274663 --0.6752620892 -float64 -int32 -[ 6. 10. 12.] -``` - -[appveyor shield]: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor - -[codecov shield]: https://img.shields.io/codecov/c/github/pythonnet/pythonnet/master.svg?label=Codecov - -[coverity shield]: https://img.shields.io/coverity/scan/7830.svg - -[license shield]: https://img.shields.io/badge/license-MIT-blue.svg?maxAge=3600 - -[pypi package version]: https://img.shields.io/pypi/v/pythonnet.svg - -[python supported shield]: https://img.shields.io/pypi/pyversions/pythonnet.svg - -[slack]: https://img.shields.io/badge/chat-slack-color.svg?style=social - -[stackexchange shield]: https://img.shields.io/badge/StackOverflow-python.net-blue.svg - -[travis shield]: https://img.shields.io/travis/pythonnet/pythonnet/master.svg?label=Travis diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..a03468e7d --- /dev/null +++ b/README.rst @@ -0,0 +1,134 @@ +pythonnet - Python.NET +=========================== + +|Join the chat at https://gitter.im/pythonnet/pythonnet| |stackexchange shield| + +|gh shield| |appveyor shield| + +|license shield| |pypi package version| |conda-forge version| |python supported shield| + +Python.NET is a package that gives Python programmers nearly +seamless integration with the .NET Common Language Runtime (CLR) and +provides a powerful application scripting tool for .NET developers. It +allows Python code to interact with the CLR, and may also be used to +embed Python into a .NET application. + +.. note:: + The master branch of this repository tracks the ongoing development of version 3.0. + Backports of patches to 2.5 are tracked in the + `backports-2.5 branch `_. + +Calling .NET code from Python +----------------------------- + +Python.NET allows CLR namespaces to be treated essentially as Python packages. + +.. code-block:: python + + import clr + from System import String + from System.Collections import * + +To load an assembly, use the ``AddReference`` function in the ``clr`` +module: + +.. code-block:: python + + import clr + clr.AddReference("System.Windows.Forms") + from System.Windows.Forms import Form + +Embedding Python in .NET +------------------------ + +- All calls to python should be inside a + ``using (Py.GIL()) {/* Your code here */}`` block. +- Import python modules using ``dynamic mod = Py.Import("mod")``, then + you can call functions as normal, eg ``mod.func(args)``. +- Use ``mod.func(args, Py.kw("keywordargname", keywordargvalue))`` or + ``mod.func(args, keywordargname: keywordargvalue)`` to apply keyword + arguments. +- All python objects should be declared as ``dynamic`` type. +- Mathematical operations involving python and literal/managed types + must have the python object first, eg. ``np.pi * 2`` works, + ``2 * np.pi`` doesn't. + +Example +~~~~~~~ + +.. code-block:: csharp + + static void Main(string[] args) + { + using (Py.GIL()) + { + dynamic np = Py.Import("numpy"); + Console.WriteLine(np.cos(np.pi * 2)); + + dynamic sin = np.sin; + Console.WriteLine(sin(5)); + + double c = np.cos(5) + sin(5); + Console.WriteLine(c); + + dynamic a = np.array(new List { 1, 2, 3 }); + Console.WriteLine(a.dtype); + + dynamic b = np.array(new List { 6, 5, 4 }, dtype: np.int32); + Console.WriteLine(b.dtype); + + Console.WriteLine(a * b); + Console.ReadKey(); + } + } + +Output: + +.. code:: csharp + + 1.0 + -0.958924274663 + -0.6752620892 + float64 + int32 + [ 6. 10. 12.] + + + +Resources +--------- + +Information on installation, FAQ, troubleshooting, debugging, and +projects using pythonnet can be found in the Wiki: + +https://github.com/pythonnet/pythonnet/wiki + +Mailing list + https://mail.python.org/mailman/listinfo/pythondotnet +Chat + https://gitter.im/pythonnet/pythonnet + +.NET Foundation +--------------- +This project is supported by the `.NET Foundation `_. + +.. |Join the chat at https://gitter.im/pythonnet/pythonnet| image:: https://badges.gitter.im/pythonnet/pythonnet.svg + :target: https://gitter.im/pythonnet/pythonnet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |appveyor shield| image:: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor + :target: https://ci.appveyor.com/project/pythonnet/pythonnet/branch/master +.. |travis shield| image:: https://img.shields.io/travis/pythonnet/pythonnet/master.svg?label=Travis + :target: https://travis-ci.org/pythonnet/pythonnet +.. |codecov shield| image:: https://img.shields.io/codecov/c/github/pythonnet/pythonnet/master.svg?label=Codecov + :target: https://codecov.io/github/pythonnet/pythonnet +.. |license shield| image:: https://img.shields.io/badge/license-MIT-blue.svg?maxAge=3600 + :target: ./LICENSE +.. |pypi package version| image:: https://img.shields.io/pypi/v/pythonnet.svg + :target: https://pypi.python.org/pypi/pythonnet +.. |python supported shield| image:: https://img.shields.io/pypi/pyversions/pythonnet.svg + :target: https://pypi.python.org/pypi/pythonnet +.. |stackexchange shield| image:: https://img.shields.io/badge/StackOverflow-python.net-blue.svg + :target: http://stackoverflow.com/questions/tagged/python.net +.. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg + :target: https://anaconda.org/conda-forge/pythonnet +.. |gh shield| image:: https://github.com/pythonnet/pythonnet/workflows/GitHub%20Actions/badge.svg + :target: https://github.com/pythonnet/pythonnet/actions?query=branch%3Amaster diff --git a/appveyor.yml b/appveyor.yml index c108801e7..1ad673ede 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,23 +1,32 @@ version: '{branch}-{build}' build: off +image: + - Visual Studio 2019 + platform: - x86 - x64 environment: global: - PYTHONUNBUFFERED: True + PYTHONUNBUFFERED: 'True' PYTHONWARNINGS: 'ignore:::wheel.pep425tags:' CODECOV_ENV: PYTHON_VERSION, PLATFORM matrix: - - PYTHON_VERSION: 2.7 - - PYTHON_VERSION: 3.3 - - PYTHON_VERSION: 3.4 - - PYTHON_VERSION: 3.5 + - PYTHON_VERSION: 3.9 + - PYTHON_VERSION: 3.8 + - PYTHON_VERSION: 3.7 - PYTHON_VERSION: 3.6 - + - PYTHON_VERSION: 3.9 + PYTHONNET_SHUTDOWN_MODE: Soft + - PYTHON_VERSION: 3.8 + PYTHONNET_SHUTDOWN_MODE: Soft + - PYTHON_VERSION: 3.7 + PYTHONNET_SHUTDOWN_MODE: Soft + - PYTHON_VERSION: 3.6 + PYTHONNET_SHUTDOWN_MODE: Soft init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% @@ -28,35 +37,20 @@ init: - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% install: + - python -m pip install -U pip - pip install --upgrade -r requirements.txt --quiet - # Install OpenCover. Can't put on `packages.config`, not Mono compatible - - .\tools\nuget\nuget.exe install OpenCover -OutputDirectory packages -Verbosity quiet - build_script: + - python setup.py configure # Create clean `sdist`. Only used for releases - python setup.py --quiet sdist - # Build `wheel` with coverage of `setup.py` - - coverage run setup.py bdist_wheel + - python setup.py bdist_wheel test_script: - pip install --no-index --find-links=.\dist\ pythonnet - - ps: .\ci\appveyor_run_tests.ps1 - - ps: .\ci\appveyor_build_recipe.ps1 - -on_finish: - # Temporary disable multiple upload due to codecov limit of 20 per commit. - # https://docs.codecov.io/blog/week-8-2017 - # - coverage xml -i - # - codecov --file coverage.xml --flags setup_windows - # - codecov --file py.coverage --flags python_tests - # - codecov --file cs.coverage --flags embedded_tests - - codecov --flags setup_windows + #- ps: .\ci\appveyor_run_tests.ps1 + - pytest + - dotnet test src/embed_tests/ artifacts: - path: dist\* - -notifications: - - provider: Slack - incoming_webhook: - secure: 2S/t6rGHdbwoxehnvn5KgfsHrBFEtwnPD7M5olGErmz70oWFVpqoWd/EvDwh7rKZGdOTjDmpwcukc2xi5VRaGHbBAqFYS3tAdgAMrcaTNWs= diff --git a/ci/appveyor_build_recipe.ps1 b/ci/appveyor_build_recipe.ps1 deleted file mode 100644 index 52520f4c2..000000000 --- a/ci/appveyor_build_recipe.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -# Build `conda.recipe` only if this is a Pull_Request. Saves time for CI. - -$env:CONDA_PY = "$env:PY_VER" -# Use pre-installed miniconda. Note that location differs if 64bit -$env:CONDA_BLD = "C:\miniconda36" - -if ($env:PLATFORM -eq "x86"){ - $env:CONDA_BLD_ARCH=32 -} else { - $env:CONDA_BLD_ARCH=64 - $env:CONDA_BLD = "$env:CONDA_BLD" + "-x64" -} - -if ($env:APPVEYOR_PULL_REQUEST_NUMBER -or $env:FORCE_CONDA_BUILD -eq "True") { - # Update PATH, and keep a copy to restore at end of this PowerShell script - $old_path = $env:path - $env:path = "$env:CONDA_BLD;$env:CONDA_BLD\Scripts;" + $env:path - - Write-Host "Starting conda install" -ForegroundColor "Green" - conda config --set always_yes True - conda config --set changeps1 False - conda config --set auto_update_conda False - conda install conda-build jinja2 anaconda-client --quiet - conda info - - # why `2>&1 | %{ "$_" }`? Redirect STDERR to STDOUT - # see: http://stackoverflow.com/a/20950421/5208670 - Write-Host "Starting conda build recipe" -ForegroundColor "Green" - conda build conda.recipe --quiet 2>&1 | %{ "$_" } - - $CONDA_PKG=(conda build conda.recipe --output) - Copy-Item $CONDA_PKG .\dist\ - Write-Host "Completed conda build recipe" -ForegroundColor "Green" - - # Restore PATH back to original - $env:path = $old_path -} else { - Write-Host "Skipping conda build recipe" -ForegroundColor "Green" -} diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index 4245d1577..7d35131f4 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -1,8 +1,13 @@ # Script to simplify AppVeyor configuration and resolve path to tools +$stopwatch = [Diagnostics.Stopwatch]::StartNew() +[array]$timings = @() + # Test Runner framework being used for embedded tests $CS_RUNNER = "nunit3-console" +$XPLAT = $env:BUILD_OPTS -eq "--xplat" + # Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3 if ($FALSE -and $env:PLATFORM -eq "x86"){ $CS_RUNNER = $CS_RUNNER + "-x86" @@ -11,13 +16,29 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){ # Executable paths for OpenCover # Note if OpenCover fails, it won't affect the exit codes. $OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe -$CS_RUNNER = Resolve-Path .\packages\NUnit.*\tools\"$CS_RUNNER".exe +if ($XPLAT){ + $CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe +} +else{ + $CS_RUNNER = Resolve-Path .\packages\NUnit.*\tools\"$CS_RUNNER".exe +} $PY = Get-Command python # Can't use ".\build\*\Python.EmbeddingTest.dll". Missing framework files. $CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll" $RUNTIME_DIR = ".\src\runtime\bin\" +function ReportTime { + param([string] $action) + + $timeSpent = $stopwatch.Elapsed + $timings += [pscustomobject]@{action=$action; timeSpent=$timeSpent} + Write-Host $action " in " $timeSpent -ForegroundColor "Green" + $stopwatch.Restart() +} + +ReportTime "Preparation done" + # Run python tests with C# coverage Write-Host ("Starting Python tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage ` @@ -26,21 +47,74 @@ Write-Host ("Starting Python tests") -ForegroundColor "Green" $PYTHON_STATUS = $LastExitCode if ($PYTHON_STATUS -ne 0) { Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Python tests completed" } # Run Embedded tests with C# coverage Write-Host ("Starting embedded tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:cs.coverage ` - -target:"$CS_RUNNER" -targetargs:"$CS_TESTS" ` + -target:"$CS_RUNNER" -targetargs:"$CS_TESTS --labels=All" ` -filter:"+[*]Python.Runtime*" ` -returntargetcode $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests failed" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Embedded tests completed" + + # NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5 + # the test is only built using modern stack + if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) { + # Run C# Performance tests + Write-Host ("Starting performance tests") -ForegroundColor "Green" + if ($XPLAT) { + $CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll" + } + else { + $CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll" + } + &"$CS_RUNNER" "$CS_PERF_TESTS" + $CS_PERF_STATUS = $LastExitCode + if ($CS_PERF_STATUS -ne 0) { + Write-Host "Performance tests (C#) failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime "Performance tests (C#) completed" + } + } else { + Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow" + Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow" + $CS_PERF_STATUS = 0 + } +} + +if ($XPLAT){ + if ($env:PLATFORM -eq "x64") { + $DOTNET_CMD = "dotnet" + } + else{ + $DOTNET_CMD = "c:\Program Files (x86)\dotnet\dotnet" + } + + # Run Embedded tests for netcoreapp3.1 (OpenCover currently does not supports dotnet core) + Write-Host ("Starting embedded tests for netcoreapp3.1") -ForegroundColor "Green" + &$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp3.1_publish\Python.EmbeddingTest.dll" + $CS_STATUS = $LastExitCode + if ($CS_STATUS -ne 0) { + Write-Host "Embedded tests for netcoreapp3.1 failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime ".NET Core 2.0 tests completed" + } } +Write-Host "Timings:" ($timings | Format-Table | Out-String) -ForegroundColor "Green" + # Set exit code to fail if either Python or Embedded tests failed -if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) { +if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) { Write-Host "Tests failed" -ForegroundColor "Red" $host.SetShouldExit(1) } diff --git a/clr.py b/clr.py new file mode 100644 index 000000000..711333dd2 --- /dev/null +++ b/clr.py @@ -0,0 +1,41 @@ +""" +Legacy Python.NET loader for backwards compatibility +""" + +def _get_netfx_path(): + import os, sys + + if sys.maxsize > 2 ** 32: + arch = "amd64" + else: + arch = "x86" + + return os.path.join(os.path.dirname(__file__), "pythonnet", "netfx", arch, "clr.pyd") + + +def _get_mono_path(): + import os, glob + + paths = glob.glob(os.path.join(os.path.dirname(__file__), "pythonnet", "mono", "clr.*so")) + return paths[0] + + +def _load_clr(): + import sys + from importlib import util + + if sys.platform == "win32": + path = _get_netfx_path() + else: + path = _get_mono_path() + + del sys.modules[__name__] + + spec = util.spec_from_file_location("clr", path) + clr = util.module_from_spec(spec) + spec.loader.exec_module(clr) + + sys.modules[__name__] = clr + + +_load_clr() diff --git a/conda.recipe/README.md b/conda.recipe/README.md deleted file mode 100644 index 42241999f..000000000 --- a/conda.recipe/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Conda Recipe - -The files here are needed to build Python.Net with conda - -http://conda.pydata.org/docs/building/recipe.html diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat deleted file mode 100644 index 1495f877d..000000000 --- a/conda.recipe/bld.bat +++ /dev/null @@ -1,6 +0,0 @@ -:: build it - -:: set path to modern MSBuild -set PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% - -%PYTHON% setup.py install diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml deleted file mode 100644 index 98602481f..000000000 --- a/conda.recipe/meta.yaml +++ /dev/null @@ -1,29 +0,0 @@ -package: - name: pythonnet - version: "2.3.0" - -build: - skip: True # [not win] - number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} - {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %}string: py{{ environ.get('PY_VER').replace('.', '') }}_0 - {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} - -source: - git_url: ../ - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - imports: - - clr - -about: - home: https://github.com/pythonnet/pythonnet - license: MIT - license_file: LICENSE diff --git a/demo/DynamicGrid.py b/demo/DynamicGrid.py new file mode 100644 index 000000000..951a6c248 --- /dev/null +++ b/demo/DynamicGrid.py @@ -0,0 +1,23 @@ +import clr +import sys +if sys.platform.lower() not in ['cli','win32']: + print("only windows is supported for wpf") +clr.AddReference(r"wpf\PresentationFramework") +from System.IO import StreamReader +from System.Windows.Markup import XamlReader +from System.Threading import Thread, ThreadStart, ApartmentState +from System.Windows import Application, Window + + +class MyWindow(Window): + def __init__(self): + stream = StreamReader("DynamicGrid.xaml") + window = XamlReader.Load(stream.BaseStream) + Application().Run(window) + + +if __name__ == '__main__': + thread = Thread(ThreadStart(MyWindow)) + thread.SetApartmentState(ApartmentState.STA) + thread.Start() + thread.Join() diff --git a/demo/DynamicGrid.xaml b/demo/DynamicGrid.xaml new file mode 100644 index 000000000..3c82eb16d --- /dev/null +++ b/demo/DynamicGrid.xaml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + diff --git a/demo/wordpad.py b/demo/wordpad.py index 876372100..409c8ad4d 100644 --- a/demo/wordpad.py +++ b/demo/wordpad.py @@ -382,7 +382,7 @@ def InitializeComponent(self): self.label1.Size = System.Drawing.Size(296, 140) self.label1.TabIndex = 2 self.label1.Text = "Python Wordpad - an example winforms " \ - "application using Python for .NET" + "application using Python.NET" self.AutoScaleBaseSize = System.Drawing.Size(5, 13) self.ClientSize = System.Drawing.Size(300, 150) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..83a58d126 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "pycparser"] +build-backend = "setuptools.build_meta" diff --git a/pythonnet.sln b/pythonnet.sln index c5afd66c3..4da4d7e99 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -1,202 +1,179 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25420.1 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.Runtime", "src\runtime\Python.Runtime.csproj", "{097B4AC0-74E9-4C58-BCF8-C69746EC8271}" +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30717.126 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime", "src\runtime\Python.Runtime.csproj", "{4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.Test", "src\testing\Python.Test.csproj", "{6F401A34-273B-450F-9A4C-13550BE0767B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console", "src\console\Console.csproj", "{E6B01706-00BA-4144-9029-186AC42FBE9A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.EmbeddingTest", "src\embed_tests\Python.EmbeddingTest.csproj", "{4165C59D-2822-499F-A6DB-EACA4C331EB5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule", "src\clrmodule\clrmodule.csproj", "{F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Console", "src\console\Console.csproj", "{E29DCF0A-5114-4A98-B1DD-71264B6EA349}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest", "src\embed_tests\Python.EmbeddingTest.csproj", "{819E089B-4770-400E-93C6-4F7A35F0EA12}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "clrmodule", "src\clrmodule\clrmodule.csproj", "{86E834DE-1139-4511-96CC-69636A56E7AC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\Python.Test.csproj", "{14EF9518-5BB7-4F83-8686-015BD2CC788E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + .gitignore = .gitignore + CHANGELOG.md = CHANGELOG.md + README.rst = README.rst + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}" + ProjectSection(SolutionItems) = preProject + appveyor.yml = appveyor.yml + ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 + ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 + .github\workflows\main.yml = .github\workflows\main.yml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" + ProjectSection(SolutionItems) = preProject + setup.py = setup.py + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}" + ProjectSection(SolutionItems) = preProject + conda.recipe\bld.bat = conda.recipe\bld.bat + conda.recipe\meta.yaml = conda.recipe\meta.yaml + conda.recipe\README.md = conda.recipe\README.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{BC426F42-8494-4AA5-82C9-5109ACD97BD1}" + ProjectSection(SolutionItems) = preProject + tools\geninterop\geninterop.py = tools\geninterop\geninterop.py + EndProjectSection +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PythonTestsRunner", "src\python_tests_runner\Python.PythonTestsRunner.csproj", "{35CBBDEB-FC07-4D04-9D3E-F88FC180110B}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution - DebugMono|x64 = DebugMono|x64 - DebugMono|x86 = DebugMono|x86 - DebugMonoPY3|x64 = DebugMonoPY3|x64 - DebugMonoPY3|x86 = DebugMonoPY3|x86 - DebugWin|x64 = DebugWin|x64 - DebugWin|x86 = DebugWin|x86 - DebugWinPY3|x64 = DebugWinPY3|x64 - DebugWinPY3|x86 = DebugWinPY3|x86 - ReleaseMono|x64 = ReleaseMono|x64 - ReleaseMono|x86 = ReleaseMono|x86 - ReleaseMonoPY3|x64 = ReleaseMonoPY3|x64 - ReleaseMonoPY3|x86 = ReleaseMonoPY3|x86 - ReleaseWin|x64 = ReleaseWin|x64 - ReleaseWin|x86 = ReleaseWin|x86 - ReleaseWinPY3|x64 = ReleaseWinPY3|x64 - ReleaseWinPY3|x86 = ReleaseWinPY3|x86 + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x64.Build.0 = DebugMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x86.Build.0 = DebugMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x64.Build.0 = DebugWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x86.Build.0 = DebugWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x64.Build.0 = DebugMono|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x86.Build.0 = DebugMono|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWin|x64.Build.0 = DebugWin|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWin|x86.Build.0 = DebugWin|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {6F401A34-273B-450F-9A4C-13550BE0767B}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMono|x64.Build.0 = DebugMono|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMono|x86.Build.0 = DebugMono|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWin|x64.Build.0 = DebugWin|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWin|x86.Build.0 = DebugWin|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {4165C59D-2822-499F-A6DB-EACA4C331EB5}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMono|x64.Build.0 = DebugMono|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMono|x86.Build.0 = DebugMono|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWin|x64.Build.0 = DebugWin|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWin|x86.Build.0 = DebugWin|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {E29DCF0A-5114-4A98-B1DD-71264B6EA349}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWin|x64.Build.0 = DebugWin|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWin|x86.Build.0 = DebugWin|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|x64.ActiveCfg = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|x64.Build.0 = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|x86.ActiveCfg = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Debug|x86.Build.0 = Debug|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|Any CPU.Build.0 = Release|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x64.ActiveCfg = Release|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x64.Build.0 = Release|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x86.ActiveCfg = Release|Any CPU + {4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}.Release|x86.Build.0 = Release|Any CPU + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|Any CPU.ActiveCfg = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|Any CPU.Build.0 = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|x64.ActiveCfg = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|x64.Build.0 = Debug|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|x86.ActiveCfg = Debug|x86 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Debug|x86.Build.0 = Debug|x86 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|Any CPU.ActiveCfg = Release|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|Any CPU.Build.0 = Release|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x64.ActiveCfg = Release|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x64.Build.0 = Release|x64 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x86.ActiveCfg = Release|x86 + {E6B01706-00BA-4144-9029-186AC42FBE9A}.Release|x86.Build.0 = Release|x86 + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Debug|x64.ActiveCfg = Debug|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Debug|x64.Build.0 = Debug|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Debug|x86.ActiveCfg = Debug|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Debug|x86.Build.0 = Debug|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Release|Any CPU.Build.0 = Release|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Release|x64.ActiveCfg = Release|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Release|x64.Build.0 = Release|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Release|x86.ActiveCfg = Release|Any CPU + {F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}.Release|x86.Build.0 = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|Any CPU.Build.0 = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|x64.ActiveCfg = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|x64.Build.0 = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|x86.ActiveCfg = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Debug|x86.Build.0 = Debug|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|Any CPU.ActiveCfg = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|Any CPU.Build.0 = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x64.ActiveCfg = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x64.Build.0 = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x86.ActiveCfg = Release|Any CPU + {819E089B-4770-400E-93C6-4F7A35F0EA12}.Release|x86.Build.0 = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|x64.ActiveCfg = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|x64.Build.0 = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|x86.ActiveCfg = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Debug|x86.Build.0 = Debug|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|Any CPU.Build.0 = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x64.ActiveCfg = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x64.Build.0 = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x86.ActiveCfg = Release|Any CPU + {14EF9518-5BB7-4F83-8686-015BD2CC788E}.Release|x86.Build.0 = Release|Any CPU + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|Any CPU.ActiveCfg = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|Any CPU.Build.0 = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|x64.ActiveCfg = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|x64.Build.0 = Debug|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|x86.ActiveCfg = Debug|x86 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Debug|x86.Build.0 = Debug|x86 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|Any CPU.ActiveCfg = Release|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|Any CPU.Build.0 = Release|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x64.ActiveCfg = Release|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x64.Build.0 = Release|x64 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.ActiveCfg = Release|x86 + {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.ActiveCfg = Debug|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.Build.0 = Debug|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x86.ActiveCfg = Debug|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x86.Build.0 = Debug|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|Any CPU.Build.0 = Release|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x64.ActiveCfg = Release|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x64.Build.0 = Release|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x86.ActiveCfg = Release|Any CPU + {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x86.Build.0 = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x64.ActiveCfg = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x64.Build.0 = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x86.ActiveCfg = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x86.Build.0 = Debug|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|Any CPU.Build.0 = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x64.ActiveCfg = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x64.Build.0 = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x86.ActiveCfg = Release|Any CPU + {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection - GlobalSection(MonoDevelopProperties) = preSolution - StartupItem = src\console\Console.csproj - Policies = $0 - $0.VersionControlPolicy = $1 - $1.inheritsSet = Mono - $0.ChangeLogPolicy = $2 - $2.UpdateMode = None - $2.MessageStyle = $3 - $3.LineAlign = 0 - $2.inheritsSet = Mono + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C8845072-C642-4858-8627-27E862AD21BB} EndGlobalSection EndGlobal diff --git a/pythonnet/.gitignore b/pythonnet/.gitignore new file mode 100644 index 000000000..7ebf7884b --- /dev/null +++ b/pythonnet/.gitignore @@ -0,0 +1,3 @@ +mono/ +netfx/ +runtime/ diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py new file mode 100644 index 000000000..5c197e146 --- /dev/null +++ b/pythonnet/__init__.py @@ -0,0 +1,3 @@ +def get_assembly_path(): + import os + return os.path.dirname(__file__) + "/runtime/Python.Runtime.dll" diff --git a/pythonnet/mono/.gitkeep b/pythonnet/mono/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/pythonnet/netfx/.gitkeep b/pythonnet/netfx/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/pythonnet/runtime/.gitkeep b/pythonnet/runtime/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/requirements.txt b/requirements.txt index bcceedf25..6f25858bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ # Requirements for both Travis and AppVeyor pytest +psutil + +# Coverage upload coverage +codecov # Platform specific requirements -pip; sys_platform == 'win32' -wheel; sys_platform == 'win32' -pycparser; sys_platform != 'win32' - -# Coverage upload -# codecov v2.0.6 isn't on PyPi -https://github.com/codecov/codecov-python/tarball/v2.0.6 +wheel +pycparser +setuptools diff --git a/setup.cfg b/setup.cfg index 38aa3eb3d..19c6f9fc9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -# [bumpversion] comments. bumpversion deleted all comments on its file. -# Don't combine `.bumpversion.cfg` with `setup.cfg`. Messes up formatting. -# Don't use `first_value = 1`. It will break `release` bump -# Keep `optional = dummy` needed to bump to release. -# See: https://github.com/peritus/bumpversion/issues/59 - [tool:pytest] xfail_strict = True # -r fsxX: show extra summary info for: (f)ailed, (s)kip, (x)failed, (X)passed diff --git a/setup.py b/setup.py index b85e2c8ad..06a26ef95 100644 --- a/setup.py +++ b/setup.py @@ -1,406 +1,285 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -Setup script for building clr.pyd and dependencies using mono and into -an egg or wheel. -""" - -import collections -import fnmatch -import glob -import os -import subprocess -import sys -import sysconfig -from distutils import spawn -from distutils.command import build_ext, install_data, install_lib - -from setuptools import Extension, setup - -# Allow config/verbosity to be set from cli -# http://stackoverflow.com/a/4792601/5208670 -CONFIG = "Release" # Release or Debug -VERBOSITY = "minimal" # quiet, minimal, normal, detailed, diagnostic - -is_64bits = sys.maxsize > 2**32 -DEVTOOLS = "MsDev" if sys.platform == "win32" else "Mono" -ARCH = "x64" if is_64bits else "x86" -PY_MAJOR = sys.version_info[0] -PY_MINOR = sys.version_info[1] -############################################################################### -# Windows Keys Constants for MSBUILD tools -RegKey = collections.namedtuple('RegKey', 'sdk_name key value_name suffix') -vs_python = "Programs\\Common\\Microsoft\\Visual C++ for Python\\9.0\\WinSDK" -vs_root = "SOFTWARE\\Microsoft\\MSBuild\\ToolsVersions\\{0}" -sdks_root = "SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v{0}Win32Tools" -kits_root = "SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots" -kits_suffix = os.path.join("bin", ARCH) +from setuptools import setup, Command, Extension +from setuptools.command.build_ext import build_ext +import distutils +from distutils.command import build +from subprocess import check_output, check_call -WIN_SDK_KEYS = ( - RegKey(sdk_name="Windows Kit 10.0", key=kits_root, - value_name="KitsRoot10", suffix=kits_suffix), +import sys, os - RegKey(sdk_name="Windows Kit 8.1", key=kits_root, - value_name="KitsRoot81", suffix=kits_suffix), +BUILD_MONO = True +BUILD_NETFX = True - RegKey(sdk_name="Windows Kit 8.0", key=kits_root, - value_name="KitsRoot", suffix=kits_suffix), +PY_MAJOR = sys.version_info[0] +PY_MINOR = sys.version_info[1] - RegKey(sdk_name="Windows SDK 7.1A", key=sdks_root.format("7.1A\\WinSDK-"), - value_name="InstallationFolder", suffix=""), +CONFIGURED_PROPS = "configured.props" - RegKey(sdk_name="Windows SDK 7.1", key=sdks_root.format("7.1\\WinSDK"), - value_name="InstallationFolder", suffix=""), - RegKey(sdk_name="Windows SDK 7.0A", key=sdks_root.format("7.0A\\WinSDK-"), - value_name="InstallationFolder", suffix=""), +def _get_interop_filename(): + """interopXX.cs is auto-generated as part of the build. + For common windows platforms pre-generated files are included + as most windows users won't have Clang installed, which is + required to generate the file. + """ + interop_filename = "interop{0}{1}{2}.cs".format( + PY_MAJOR, PY_MINOR, getattr(sys, "abiflags", "") + ) + return os.path.join("src", "runtime", interop_filename) - RegKey(sdk_name="Windows SDK 7.0", key=sdks_root.format("7.0\\WinSDK"), - value_name="InstallationFolder", suffix=""), - RegKey(sdk_name="Windows SDK 6.0A", key=sdks_root.format("6.0A\\WinSDK"), - value_name="InstallationFolder", suffix=""), -) +# Write configuration to configured.props. This will go away once all of these +# can be decided at runtime. +def _write_configure_props(): + defines = [ + "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR), + ] -VS_KEYS = ( - RegKey(sdk_name="MSBuild 14", key=vs_root.format("14.0"), - value_name="MSBuildToolsPath", suffix=""), + if sys.platform == "win32": + defines.append("WINDOWS") - RegKey(sdk_name="MSBuild 12", key=vs_root.format("12.0"), - value_name="MSBuildToolsPath", suffix=""), + if hasattr(sys, "abiflags"): + if "d" in sys.abiflags: + defines.append("PYTHON_WITH_PYDEBUG") + if "m" in sys.abiflags: + defines.append("PYTHON_WITH_PYMALLOC") - RegKey(sdk_name="MSBuild 4", key=vs_root.format("4.0"), - value_name="MSBuildToolsPath", suffix=""), + # check the interop file exists, and create it if it doesn't + interop_file = _get_interop_filename() + if not os.path.exists(interop_file): + print("Creating {0}".format(interop_file)) + geninterop = os.path.join("tools", "geninterop", "geninterop.py") + check_call([sys.executable, geninterop, interop_file]) - RegKey(sdk_name="MSBuild 3.5", key=vs_root.format("3.5"), - value_name="MSBuildToolsPath", suffix=""), + import xml.etree.ElementTree as ET - RegKey(sdk_name="MSBuild 2.0", key=vs_root.format("2.0"), - value_name="MSBuildToolsPath", suffix=""), -) + proj = ET.Element("Project") + props = ET.SubElement(proj, "PropertyGroup") + f = ET.SubElement(props, "PythonInteropFile") + f.text = os.path.basename(interop_file) + c = ET.SubElement(props, "ConfiguredConstants") + c.text = " ".join(defines) -############################################################################### -def _check_output(*args, **kwargs): - """Check output wrapper for py2/py3 compatibility""" - output = subprocess.check_output(*args, **kwargs) - if PY_MAJOR == 2: - return output - return output.decode("ascii") + ET.ElementTree(proj).write(CONFIGURED_PROPS) -def _get_interop_filename(): - """interopXX.cs is auto-generated as part of the build. - For common windows platforms pre-generated files are included - as most windows users won't have Clang installed, which is - required to generate the file. - """ - interop_filename = "interop{0}{1}{2}.cs".format( - PY_MAJOR, PY_MINOR, getattr(sys, "abiflags", "")) - return os.path.join("src", "runtime", interop_filename) - - -def _get_source_files(): - """Walk project and collect the files needed for ext_module""" - for ext in (".sln", ): - for path in glob.glob("*" + ext): - yield path +class configure(Command): + """Configure command""" - for root, dirnames, filenames in os.walk("src"): - for ext in (".cs", ".csproj", ".snk", ".config", - ".py", ".c", ".h", ".ico"): - for filename in fnmatch.filter(filenames, "*" + ext): - yield os.path.join(root, filename) + description = "Configure the pythonnet build" + user_options = [] - for root, dirnames, filenames in os.walk("tools"): - for ext in (".exe", ".py", ".c", ".h"): - for filename in fnmatch.filter(filenames, "*" + ext): - yield os.path.join(root, filename) + def initialize_options(self): + pass + def finalize_options(self): + pass -def _get_long_description(): - """Helper to populate long_description for pypi releases""" - try: - import pypandoc - return pypandoc.convert('README.md', 'rst') - except ImportError: - return '.Net and Mono integration for Python' - - -class BuildExtPythonnet(build_ext.build_ext): - def build_extension(self, ext): - """Builds the .pyd file using msbuild or xbuild""" - if ext.name != "clr": - return build_ext.build_ext.build_extension(self, ext) - - # install packages using nuget - self._install_packages() - - dest_file = self.get_ext_fullpath(ext.name) - dest_dir = os.path.dirname(dest_file) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - - # Up to Python 3.2 sys.maxunicode is used to determine the size of - # Py_UNICODE, but from 3.3 onwards Py_UNICODE is a typedef of wchar_t. - # TODO: Is this doing the right check for Py27? - if sys.version_info[:2] <= (3, 2): - unicode_width = 2 if sys.maxunicode < 0x10FFFF else 4 + def run(self): + self.announce("Writing configured.props...", level=distutils.log.INFO) + _write_configure_props() + + +class DotnetLib: + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + self.args = kwargs + + +class build_dotnet(Command): + """Build command for dotnet-cli based builds""" + + description = "Build DLLs with dotnet-cli" + user_options = [ + ("dotnet-config", None, "dotnet build configuration"), + ( + "inplace", + "i", + "ignore build-lib and put compiled extensions into the source " + + "directory alongside your pure Python modules", + ), + ] + + def initialize_options(self): + self.dotnet_config = None + self.build_lib = None + self.inplace = False + + def finalize_options(self): + if self.dotnet_config is None: + self.dotnet_config = "release" + + build = self.distribution.get_command_obj("build") + build.ensure_finalized() + if self.inplace: + self.build_lib = "." else: - import ctypes - unicode_width = ctypes.sizeof(ctypes.c_wchar) + self.build_lib = build.build_lib - defines = [ - "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR), - "PYTHON{0}".format(PY_MAJOR), # Python Major Version - "UCS{0}".format(unicode_width), + def run(self): + dotnet_modules = self.distribution.dotnet_libs + self.run_command("configure") + + for lib in dotnet_modules: + output = os.path.join( + os.path.abspath(self.build_lib), lib.args.pop("output") + ) + rename = lib.args.pop("rename", {}) + + opts = sum( + [ + ["--" + name.replace("_", "-"), value] + for name, value in lib.args.items() + ], + [], + ) + + opts.extend(["--configuration", self.dotnet_config]) + opts.extend(["--output", output]) + + self.announce("Running dotnet build...", level=distutils.log.INFO) + self.spawn(["dotnet", "build", lib.path] + opts) + + for k, v in rename.items(): + source = os.path.join(output, k) + dest = os.path.join(output, v) + + if os.path.isfile(source): + try: + os.remove(dest) + except OSError: + pass + + self.move_file(src=source, dst=dest, level=distutils.log.INFO) + else: + self.warn( + "Can't find file to rename: {}, current dir: {}".format( + source, os.getcwd() + ) + ) + + +# Add build_dotnet to the build tasks: +from distutils.command.build import build as _build +from setuptools.command.develop import develop as _develop +from setuptools import Distribution +import setuptools + + +class build(_build): + sub_commands = _build.sub_commands + [("build_dotnet", None)] + + +class develop(_develop): + def install_for_development(self): + # Build extensions in-place + self.reinitialize_command("build_dotnet", inplace=1) + self.run_command("build_dotnet") + + return super().install_for_development() + + +# Monkey-patch Distribution s.t. it supports the dotnet_libs attribute +Distribution.dotnet_libs = None + +cmdclass = { + "build": build, + "build_dotnet": build_dotnet, + "configure": configure, + "develop": develop, +} + + +with open("README.rst", "r") as f: + long_description = f.read() + +dotnet_libs = [ + DotnetLib( + "python-runtime", + "src/runtime/Python.Runtime.csproj", + output="pythonnet/runtime", + ) +] + +if BUILD_NETFX: + dotnet_libs.extend( + [ + DotnetLib( + "clrmodule-amd64", + "src/clrmodule/", + runtime="win-x64", + output="pythonnet/netfx/amd64", + rename={"clr.dll": "clr.pyd"}, + ), + DotnetLib( + "clrmodule-x86", + "src/clrmodule/", + runtime="win-x86", + output="pythonnet/netfx/x86", + rename={"clr.dll": "clr.pyd"}, + ), ] + ) - if CONFIG == "Debug": - defines.extend(["DEBUG", "TRACE"]) - - if sys.platform != "win32" and DEVTOOLS == "Mono": - on_darwin = sys.platform == "darwin" - defines.append("MONO_OSX" if on_darwin else "MONO_LINUX") - - # Check if --enable-shared was set when Python was built - enable_shared = sysconfig.get_config_var("Py_ENABLE_SHARED") - if enable_shared: - # Double-check if libpython is linked dynamically with python - ldd_cmd = ["otool", "-L"] if on_darwin else ["ldd"] - lddout = _check_output(ldd_cmd + [sys.executable]) - if 'libpython' not in lddout: - enable_shared = False - - if not enable_shared: - defines.append("PYTHON_WITHOUT_ENABLE_SHARED") - - if hasattr(sys, "abiflags"): - if "d" in sys.abiflags: - defines.append("PYTHON_WITH_PYDEBUG") - if "m" in sys.abiflags: - defines.append("PYTHON_WITH_PYMALLOC") - - # check the interop file exists, and create it if it doesn't - interop_file = _get_interop_filename() - if not os.path.exists(interop_file): - self.debug_print("Creating {0}".format(interop_file)) - geninterop = os.path.join("tools", "geninterop", "geninterop.py") - subprocess.check_call([sys.executable, geninterop, interop_file]) - - if DEVTOOLS == "MsDev": - _xbuild = '"{0}"'.format(self._find_msbuild_tool("msbuild.exe")) - _config = "{0}Win".format(CONFIG) - - elif DEVTOOLS == "Mono": - _xbuild = "xbuild" - _config = "{0}Mono".format(CONFIG) - else: - raise NotImplementedError( - "DevTool {0} not supported (use MsDev/Mono)".format(DEVTOOLS)) - - cmd = [ - _xbuild, - 'pythonnet.sln', - '/p:Configuration={}'.format(_config), - '/p:Platform={}'.format(ARCH), - '/p:DefineConstants="{}"'.format(','.join(defines)), - '/p:PythonBuildDir="{}"'.format(os.path.abspath(dest_dir)), - '/p:PythonInteropFile="{}"'.format(os.path.basename(interop_file)), - '/verbosity:{}'.format(VERBOSITY), - ] +ext_modules = [] - manifest = self._get_manifest(dest_dir) - if manifest: - cmd.append('/p:PythonManifest="{0}"'.format(manifest)) - - self.debug_print("Building: {0}".format(" ".join(cmd))) - use_shell = True if DEVTOOLS == "Mono" else False - subprocess.check_call(" ".join(cmd + ["/t:Clean"]), shell=use_shell) - subprocess.check_call(" ".join(cmd + ["/t:Build"]), shell=use_shell) - - if DEVTOOLS == "Mono": - self._build_monoclr() - - def _get_manifest(self, build_dir): - if DEVTOOLS != "MsDev": - return - mt = self._find_msbuild_tool("mt.exe", use_windows_sdk=True) - manifest = os.path.abspath(os.path.join(build_dir, "app.manifest")) - cmd = [mt, '-inputresource:"{0}"'.format(sys.executable), - '-out:"{0}"'.format(manifest)] - self.debug_print("Extracting manifest from {}".format(sys.executable)) - subprocess.check_call(" ".join(cmd), shell=False) - return manifest - - def _build_monoclr(self): - mono_libs = _check_output("pkg-config --libs mono-2", shell=True) - mono_cflags = _check_output("pkg-config --cflags mono-2", shell=True) - glib_libs = _check_output("pkg-config --libs glib-2.0", shell=True) - glib_cflags = _check_output("pkg-config --cflags glib-2.0", shell=True) - cflags = mono_cflags.strip() + " " + glib_cflags.strip() - libs = mono_libs.strip() + " " + glib_libs.strip() +if BUILD_MONO: + try: + mono_libs = check_output( + "pkg-config --libs mono-2", shell=True, encoding="utf8" + ) + mono_cflags = check_output( + "pkg-config --cflags mono-2", shell=True, encoding="utf8" + ) + cflags = mono_cflags.strip() + libs = mono_libs.strip() # build the clr python module clr_ext = Extension( - "clr", - sources=[ - "src/monoclr/pynetinit.c", - "src/monoclr/clrmod.c" - ], + "pythonnet.mono.clr", + language="c++", + sources=["src/monoclr/clrmod.c"], extra_compile_args=cflags.split(" "), - extra_link_args=libs.split(" ") + extra_link_args=libs.split(" "), + ) + ext_modules.append(clr_ext) + except Exception: + print( + "Failed to find mono libraries via pkg-config, skipping the Mono CLR loader" ) - build_ext.build_ext.build_extension(self, clr_ext) - - def _install_packages(self): - """install packages using nuget""" - nuget = os.path.join("tools", "nuget", "nuget.exe") - use_shell = False - if DEVTOOLS == "Mono": - nuget = "mono {0}".format(nuget) - use_shell = True - - cmd = "{0} update -self".format(nuget) - self.debug_print("Updating NuGet: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) - - cmd = "{0} restore pythonnet.sln -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) - - def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): - """Return full path to one of the Microsoft build tools""" - # Search in PATH first - path = spawn.find_executable(tool) - if path: - return path - - # Search within registry to find build tools - try: # PY2 - import _winreg as winreg - except ImportError: # PY3 - import winreg - - keys_to_check = WIN_SDK_KEYS if use_windows_sdk else VS_KEYS - hklm = winreg.HKEY_LOCAL_MACHINE - for rkey in keys_to_check: - try: - with winreg.OpenKey(hklm, rkey.key) as hkey: - val, type_ = winreg.QueryValueEx(hkey, rkey.value_name) - if type_ != winreg.REG_SZ: - continue - path = os.path.join(val, rkey.suffix, tool) - if os.path.exists(path): - self.debug_print("Using {0} from {1}".format( - tool, rkey.sdk_name)) - return path - except WindowsError: - # Key doesn't exist - pass - - # Add Visual C++ for Python as a fall-back in case one - # of the other Windows SDKs isn't installed. - # TODO: Extend checking by using setuptools/msvc.py? - if use_windows_sdk: - sdk_name = "Visual C++ for Python" - localappdata = os.environ["LOCALAPPDATA"] - suffix = "Bin\\x64" if ARCH == "x64" else "Bin" - path = os.path.join(localappdata, vs_python, suffix, tool) - if os.path.exists(path): - self.debug_print("Using {0} from {1}".format(tool, sdk_name)) - return path - - raise RuntimeError("{0} could not be found".format(tool)) - - -class InstallLibPythonnet(install_lib.install_lib): - def install(self): - if not os.path.isdir(self.build_dir): - self.warn("'{0}' does not exist -- no Python modules" - " to install".format(self.build_dir)) - return - - if not os.path.exists(self.install_dir): - self.mkpath(self.install_dir) - - # only copy clr.pyd/.so - for srcfile in glob.glob(os.path.join(self.build_dir, "clr.*")): - destfile = os.path.join( - self.install_dir, os.path.basename(srcfile)) - self.copy_file(srcfile, destfile) - - -class InstallDataPythonnet(install_data.install_data): - def run(self): - build_cmd = self.get_finalized_command("build_ext") - install_cmd = self.get_finalized_command("install") - build_lib = os.path.abspath(build_cmd.build_lib) - install_platlib = os.path.relpath( - install_cmd.install_platlib, self.install_dir) - - for i, data_files in enumerate(self.data_files): - if isinstance(data_files, str): - self.data_files[i] = data_files[i].format(build_lib=build_lib) - else: - for j, filename in enumerate(data_files[1]): - data_files[1][j] = filename.format(build_lib=build_lib) - dest = data_files[0].format(install_platlib=install_platlib) - self.data_files[i] = dest, data_files[1] - - return install_data.install_data.run(self) - - -############################################################################### -setupdir = os.path.dirname(__file__) -if setupdir: - os.chdir(setupdir) - -setup_requires = [] -if not os.path.exists(_get_interop_filename()): - setup_requires.append("pycparser") setup( + cmdclass=cmdclass, name="pythonnet", - version="2.3.0", + version="3.0.0.dev1", description=".Net and Mono integration for Python", - url='https://pythonnet.github.io/', - license='MIT', - author="The Python for .Net developers", - author_email="pythondotnet@python.org", - setup_requires=setup_requires, - long_description=_get_long_description(), - ext_modules=[ - Extension("clr", sources=list(_get_source_files())) - ], - data_files=[ - ("{install_platlib}", [ - "{build_lib}/Python.Runtime.dll", - ]), - ], - cmdclass={ - "build_ext": BuildExtPythonnet, - "install_lib": InstallLibPythonnet, - "install_data": InstallDataPythonnet, - }, + url="https://pythonnet.github.io/", + license="MIT", + author="The Contributors of the Python.NET Project", + author_email="pythonnet@python.org", + packages=["pythonnet"], + install_requires=["pycparser"], + long_description=long_description, + # data_files=[("{install_platlib}", ["{build_lib}/pythonnet"])], + py_modules=["clr"], + ext_modules=ext_modules, + dotnet_libs=dotnet_libs, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: C#', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX :: Linux', - 'Operating System :: MacOS :: MacOS X', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: C#", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", ], zip_safe=False, ) diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs deleted file mode 100644 index 24ba26862..000000000 --- a/src/SharedAssemblyInfo.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Reflection; -using System.Resources; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("pythonnet")] -[assembly: AssemblyProduct("Python for .NET")] -[assembly: AssemblyCopyright("Copyright (c) 2006-2017 the contributors of the 'Python for .NET' project")] -[assembly: AssemblyTrademark("")] - -[assembly: AssemblyCulture("")] -[assembly: NeutralResourcesLanguage("")] - -[assembly: CLSCompliant(true)] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// Version Information. Keeping it simple. May need to revisit for Nuget -// See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ -// AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.3.0")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index d24376a1f..7b0387d46 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -26,17 +26,12 @@ using System.IO; using System.Reflection; using System.Runtime.InteropServices; -using RGiesecke.DllExport; +using NXPorts.Attributes; public class clrModule { -#if PYTHON3 [DllExport("PyInit_clr", CallingConvention.StdCall)] public static IntPtr PyInit_clr() -#elif PYTHON2 - [DllExport("initclr", CallingConvention.StdCall)] - public static void initclr() -#endif { DebugPrint("Attempting to load 'Python.Runtime' using standard binding rules."); #if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN @@ -53,7 +48,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.3.0"), + Version = new Version("2.5.0"), #endif CultureInfo = CultureInfo.InvariantCulture }; @@ -95,11 +90,7 @@ public static void initclr() catch (InvalidOperationException) { DebugPrint("Could not load 'Python.Runtime'."); -#if PYTHON3 return IntPtr.Zero; -#elif PYTHON2 - return; -#endif } } @@ -107,11 +98,7 @@ public static void initclr() // So now we get the PythonEngine and execute the InitExt method on it. Type pythonEngineType = pythonRuntime.GetType("Python.Runtime.PythonEngine"); -#if PYTHON3 return (IntPtr)pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); -#elif PYTHON2 - pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); -#endif } /// diff --git a/src/clrmodule/Properties/AssemblyInfo.cs b/src/clrmodule/Properties/AssemblyInfo.cs index 939f4171f..5e2e05ed4 100644 --- a/src/clrmodule/Properties/AssemblyInfo.cs +++ b/src/clrmodule/Properties/AssemblyInfo.cs @@ -1,11 +1,5 @@ using System.Reflection; using System.Runtime.InteropServices; -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("clrmodule")] -[assembly: AssemblyDescription("")] - // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("ae10d6a4-55c2-482f-9716-9988e6c169e3")] diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj index 6e5ff4966..8595fd0ba 100644 --- a/src/clrmodule/clrmodule.csproj +++ b/src/clrmodule/clrmodule.csproj @@ -1,95 +1,24 @@ - - + - Debug - AnyCPU - {86E834DE-1139-4511-96CC-69636A56E7AC} - Library - clrmodule - clrmodule - bin\clrmodule.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - prompt + net472 + win-x86;win-x64 + clr - + + + + + + 1.0.0 + all + runtime; build; native; contentfiles; analyzers + + + + x86 - + x64 - - true - PYTHON2;TRACE;DEBUG - full - - - PYTHON2 - true - pdbonly - - - true - PYTHON2;TRACE;DEBUG - full - - - PYTHON2 - true - pdbonly - - - true - PYTHON3;TRACE;DEBUG - full - - - PYTHON3 - true - pdbonly - - - true - PYTHON3;TRACE;DEBUG - full - - - PYTHON3 - true - pdbonly - - - - ..\..\packages\UnmanagedExports.1.2.7\lib\net\RGiesecke.DllExport.Metadata.dll - False - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - diff --git a/src/clrmodule/packages.config b/src/clrmodule/packages.config deleted file mode 100644 index 2a95dc54d..000000000 --- a/src/clrmodule/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/src/console/Console.csproj b/src/console/Console.csproj index ea88b6356..08854cfc9 100644 --- a/src/console/Console.csproj +++ b/src/console/Console.csproj @@ -1,101 +1,27 @@ - - + - Debug - AnyCPU - {E29DCF0A-5114-4A98-B1DD-71264B6EA349} + net472;netcoreapp3.1 + x64;x86 Exe nPython Python.Runtime - bin\nPython.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 + nPython python-clear.ico - prompt - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - $(PythonManifest) - - - - - - - - Properties\SharedAssemblyInfo.cs - - - + - + Python.Runtime.dll - - {097b4ac0-74e9-4c58-bcf8-c69746ec8271} - Python.Runtime - + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + - - - - diff --git a/src/console/Properties/AssemblyInfo.cs b/src/console/Properties/AssemblyInfo.cs deleted file mode 100644 index 081ae0c94..000000000 --- a/src/console/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System.Reflection; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Python Console")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyDefaultAlias("python.exe")] diff --git a/src/console/pythonconsole.cs b/src/console/pythonconsole.cs index e9bb31e69..912e9bb0d 100644 --- a/src/console/pythonconsole.cs +++ b/src/console/pythonconsole.cs @@ -16,8 +16,9 @@ namespace Python.Runtime /// public sealed class PythonConsole { +#if NET40 private static AssemblyLoader assemblyLoader = new AssemblyLoader(); - +#endif private PythonConsole() { } @@ -25,9 +26,11 @@ private PythonConsole() [STAThread] public static int Main(string[] args) { + // Only net40 is capable to safely inject python.runtime.dll into resources. +#if NET40 // reference the static assemblyLoader to stop it being optimized away AssemblyLoader a = assemblyLoader; - +#endif string[] cmd = Environment.GetCommandLineArgs(); PythonEngine.Initialize(); @@ -37,6 +40,7 @@ public static int Main(string[] args) return i; } +#if NET40 // Register a callback function to load embedded assemblies. // (Python.Runtime.dll is included as a resource) private sealed class AssemblyLoader @@ -73,5 +77,6 @@ public AssemblyLoader() }; } } +#endif } } diff --git a/src/domain_tests/App.config b/src/domain_tests/App.config new file mode 100644 index 000000000..56efbc7b5 --- /dev/null +++ b/src/domain_tests/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj new file mode 100644 index 000000000..54196f210 --- /dev/null +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -0,0 +1,26 @@ + + + + net472 + bin\ + Exe + + + + + + + + + + + + + + + + + + + + diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs new file mode 100644 index 000000000..a21297829 --- /dev/null +++ b/src/domain_tests/TestRunner.cs @@ -0,0 +1,1321 @@ +// We can't refer to or use Python.Runtime here. +// We want it to be loaded only inside the subdomains +using System; +using Microsoft.CSharp; +using System.CodeDom.Compiler; +using System.IO; +using System.Linq; + +namespace Python.DomainReloadTests +{ + /// + /// This class provides an executable that can run domain reload tests. + /// The setup is a bit complicated: + /// 1. pytest runs test_*.py in this directory. + /// 2. test_classname runs Python.DomainReloadTests.exe (this class) with an argument + /// 3. This class at runtime creates a directory that has both C# and + /// python code, and compiles the C#. + /// 4. This class then runs the C# code. + /// + /// But there's a bit more indirection. This class compiles a DLL that + /// contains code that will change. + /// Then, the test case: + /// * Compiles some code, loads it into a domain, runs python that refers to it. + /// * Unload the domain, re-runs the domain to make sure domain reload happens correctly. + /// * Compile a new piece of code, load it into a new domain, run a new piece of + /// Python code to test the objects after they've been deleted or modified in C#. + /// * Unload the domain. Reload the domain, run the same python again. + /// + /// This class gets built into an executable which takes one argument: + /// which test case to run. That's because pytest assumes we'll run + /// everything in one process, but we really want a clean process on each + /// test case to test the init/reload/teardown parts of the domain reload. + /// + /// + class TestRunner + { + const string TestAssemblyName = "DomainTests"; + + class TestCase + { + /// + /// The key to pass as an argument to choose this test. + /// + public string Name; + + /// + /// The C# code to run in the first domain. + /// + public string DotNetBefore; + + /// + /// The C# code to run in the second domain. + /// + public string DotNetAfter; + + /// + /// The Python code to run as a module that imports the C#. + /// It should have two functions: before_reload() and after_reload(). + /// Before will be called twice when DotNetBefore is loaded; + /// after will also be called twice when DotNetAfter is loaded. + /// To make the test fail, have those functions raise exceptions. + /// + /// Make sure there's no leading spaces since Python cares. + /// + public string PythonCode; + } + + static TestCase[] Cases = new TestCase[] + { + new TestCase + { + Name = "class_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Before { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class After { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Before + + +def after_reload(): + assert sys.my_cls is not None + try: + foo = TestNamespace.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "static_member_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public static int Before() { return 5; } } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public static int After() { return 10; } } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + if not hasattr(sys, 'my_cls'): + sys.my_cls = TestNamespace.Cls + sys.my_fn = TestNamespace.Cls.Before + assert 5 == sys.my_fn() + assert 5 == TestNamespace.Cls.Before() + +def after_reload(): + + # We should have reloaded the class so we can access the new function. + assert 10 == sys.my_cls.After() + assert True is True + + try: + # We should have reloaded the class. The old function still exists, but is now invalid. + sys.my_cls.Before() + except AttributeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + assert sys.my_fn is not None + + try: + # Unbound functions still exist. They will error out when called though. + sys.my_fn() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') + ", + }, + + + new TestCase + { + Name = "member_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public int Before() { return 5; } } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public int After() { return 10; } } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls() + sys.my_fn = TestNamespace.Cls().Before + sys.my_fn() + TestNamespace.Cls().Before() + +def after_reload(): + + # We should have reloaded the class so we can access the new function. + assert 10 == sys.my_cls.After() + assert True is True + + try: + # We should have reloaded the class. The old function still exists, but is now invalid. + sys.my_cls.Before() + except AttributeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + assert sys.my_fn is not None + + try: + # Unbound functions still exist. They will error out when called though. + sys.my_fn() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') + ", + }, + + new TestCase + { + Name = "field_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Before = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int After = 4; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_int = Cls.Before + +def after_reload(): + print(sys.my_int) + try: + assert 2 == Cls.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') +", + }, + new TestCase + { + Name = "property_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Before { get { return 2; } } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int After { get { return 4; } } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_int = Cls.Before + +def after_reload(): + print(sys.my_int) + try: + assert 2 == Cls.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') +", + }, + + new TestCase + { + Name = "event_rename", + DotNetBefore = @" + using System; + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static event Action Before; + public static void Call() + { + Before(); + } + } + }", + DotNetAfter = @" + using System; + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static event Action After; + public static void Call() + { + After(); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +called = False + +def callback_function(): + global called + called = True + +def before_reload(): + global called + called = False + Cls.Before += callback_function + Cls.Call() + assert called is True + +def after_reload(): + global called + assert called is True + called = False + Cls.Call() + assert called is False +", + }, + + new TestCase + { + Name = "namespace_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public int Foo; + public Cls(int i) + { + Foo = i; + } + } + }", + DotNetAfter = @" + namespace NewTestNamespace + { + [System.Serializable] + public class Cls + { + public int Foo; + public Cls(int i) + { + Foo = i; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + sys.my_inst = TestNamespace.Cls(1) + +def after_reload(): + try: + TestNamespace.Cls(2) + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "field_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo = 1; + public static int Field = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo = 1; + private static int Field = 2; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + assert 2 == Cls.Field + assert 1 == Cls.Foo + +def after_reload(): + assert 1 == Cls.Foo + try: + assert 1 == Cls.Field + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "method_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo() { return 1; } + public static int Function() { return 2; } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo() { return 1; } + private static int Function() { return 2; } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_func = Cls.Function + assert 1 == Cls.Foo() + assert 2 == Cls.Function() + +def after_reload(): + assert 1 == Cls.Foo() + try: + assert 2 == Cls.Function() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + try: + assert 2 == sys.my_func() + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "property_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo { get { return 1; } } + public static int Property { get { return 2; } } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo { get { return 1; } } + private static int Property { get { return 2; } } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + assert 1 == Cls.Foo + assert 2 == Cls.Property + +def after_reload(): + assert 1 == Cls.Foo + try: + assert 2 == Cls.Property + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "class_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class PublicClass { } + + [System.Serializable] + public class Cls { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + internal class Cls { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + +def after_reload(): + sys.my_cls() + + try: + TestNamespace.Cls() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "method_parameters_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static void MyFunction(int a) + { + System.Console.WriteLine(string.Format(""MyFunction says: {0}"", a)); + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static void MyFunction(string a) + { + System.Console.WriteLine(string.Format(""MyFunction says: {0}"", a)); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + sys.my_func = Cls.MyFunction + sys.my_cls.MyFunction(1) + sys.my_func(2) + +def after_reload(): + try: + sys.my_cls.MyFunction(1) + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + try: + sys.my_func(2) + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + # Calling the function from the class passes + sys.my_cls.MyFunction('test') + + try: + # calling the callable directly fails + sys.my_func('test') + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + Cls.MyFunction('another test') + + ", + }, + + new TestCase + { + Name = "method_return_type_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int MyFunction() + { + return 2; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static string MyFunction() + { + return ""22""; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + sys.my_func = Cls.MyFunction + assert 2 == sys.my_cls.MyFunction() + assert 2 == sys.my_func() + +def after_reload(): + assert '22' == sys.my_cls.MyFunction() + assert '22' == sys.my_func() + assert '22' == Cls.MyFunction() + ", + }, + + new TestCase + { + Name = "field_type_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Field = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public string Field = ""22""; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + assert 2 == sys.my_cls.Field + +def after_reload(): + assert '22' == Cls.Field + assert '22' == sys.my_cls.Field + ", + }, + + new TestCase + { + Name = "construct_removed_class", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Before { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class After { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Before + +def after_reload(): + bar = sys.my_cls() + + # Don't crash! + print(bar) + print(bar.__str__()) + print(bar.__repr__()) + ", + }, + + new TestCase + { + Name = "out_to_ref_param", + DotNetBefore = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (out Data a) + { + a = new Data(); + a.num = 9001; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + assert bar.num == 9001 + # foo shouldn't have changed. + assert foo.num == -1 + + +def after_reload(): + + try: + # Now that the function takes a ref type, we must pass a valid object. + bar = TestNamespace.Cls.MyFn(None) + except System.NullReferenceException as e: + print('caught expected exception') + else: + raise AssertionError('failed to raise') + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + # Pythonnet also returns a new object with `ref`-qualified parameters + assert foo is not bar + ", + }, + + new TestCase + { + Name = "ref_to_out_param", + DotNetBefore = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (out Data a) + { + a = new Data(); + a.num = 9001; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + + +def after_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + assert bar.num == 9001 + # foo shouldn't have changed. + assert foo.num == -1 + # this should work too + baz = TestNamespace.Cls.MyFn(None) + assert baz.num == 9001 + ", + }, + new TestCase + { + Name = "ref_to_in_param", + DotNetBefore = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + System.Console.Write(""Method with ref parameter: ""); + System.Console.WriteLine(a.num); + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (Data a) + { + System.Console.Write(""Method with in parameter: ""); + System.Console.WriteLine(a.num); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + +def after_reload(): + + foo = TestNamespace.Data() + TestNamespace.Cls.MyFn(foo) + # foo should not have changed + assert foo.num == TestNamespace.Data().num + + ", + }, + new TestCase + { + Name = "in_to_ref_param", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (Data a) + { + System.Console.Write(""Method with in parameter: ""); + System.Console.WriteLine(a.num); + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + System.Console.Write(""Method with ref parameter: ""); + System.Console.WriteLine(a.num); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + TestNamespace.Cls.MyFn(foo) + # foo should not have changed + assert foo.num == TestNamespace.Data().num + +def after_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + ", + }, + new TestCase + { + Name = "nested_type", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Inner + { + public static int Value = -1; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Inner + { + public static int Value = -1; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + + sys.my_obj = TestNamespace.WithNestedType + +def after_reload(): + + assert sys.my_obj is not None + foo = sys.my_obj.Inner() + print(foo) + + ", + }, + }; + + /// + /// The runner's code. Runs the python code + /// This is a template for string.Format + /// Arg 0 is the reload mode: ShutdownMode.Reload or other. + /// Arg 1 is the no-arg python function to run, before or after. + /// + const string CaseRunnerTemplate = @" +using System; +using System.IO; +using Python.Runtime; +namespace CaseRunner +{{ + class CaseRunner + {{ + public static int Main() + {{ + try + {{ + PythonEngine.Initialize(mode:{0}); + using (Py.GIL()) + {{ + var temp = AppDomain.CurrentDomain.BaseDirectory; + dynamic sys = Py.Import(""sys""); + sys.path.append(new PyString(temp)); + dynamic test_mod = Py.Import(""domain_test_module.mod""); + test_mod.{1}_reload(); + }} + PythonEngine.Shutdown(); + }} + catch (PythonException pe) + {{ + throw new ArgumentException(message:pe.Message+"" ""+pe.StackTrace); + }} + catch (Exception e) + {{ + Console.WriteLine(e.StackTrace); + throw; + }} + return 0; + }} + }} +}} +"; + readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Python.Runtime.dll"); + + static string TestPath = null; + + public static int Main(string[] args) + { + TestCase testCase; + if (args.Length < 1) + { + testCase = Cases[0]; + } + else + { + string testName = args[0]; + Console.WriteLine($"-- Looking for domain reload test case {testName}"); + testCase = Cases.First(c => c.Name == testName); + } + + Console.WriteLine($"-- Running domain reload test case: {testCase.Name}"); + + SetupTestFolder(testCase.Name); + + CreatePythonModule(testCase); + { + var runnerAssembly = CreateCaseRunnerAssembly(verb:"before"); + CreateTestClassAssembly(testCase.DotNetBefore); + { + var runnerDomain = CreateDomain("case runner before"); + RunAndUnload(runnerDomain, runnerAssembly); + } + { + var runnerDomain = CreateDomain("case runner before (again)"); + RunAndUnload(runnerDomain, runnerAssembly); + } + } + + { + var runnerAssembly = CreateCaseRunnerAssembly(verb:"after"); + CreateTestClassAssembly(testCase.DotNetAfter); + + // Do it twice for good measure + { + var runnerDomain = CreateDomain("case runner after"); + RunAndUnload(runnerDomain, runnerAssembly); + } + { + var runnerDomain = CreateDomain("case runner after (again)"); + RunAndUnload(runnerDomain, runnerAssembly); + } + } + + // Don't delete unconditionally. It's sometimes useful to leave the + // folder behind to debug failing tests. + TeardownTestFolder(); + + return 0; + } + + static void SetupTestFolder(string testCaseName) + { + var pid = System.Diagnostics.Process.GetCurrentProcess().Id; + TestPath = Path.Combine(Path.GetTempPath(), $"Python.TestRunner.{testCaseName}-{pid}"); + if (Directory.Exists(TestPath)) + { + Directory.Delete(TestPath, recursive: true); + } + Directory.CreateDirectory(TestPath); + Console.WriteLine($"Using directory: {TestPath}"); + File.Copy(PythonDllLocation, Path.Combine(TestPath, "Python.Runtime.dll")); + } + + static void TeardownTestFolder() + { + if (Directory.Exists(TestPath)) + { + Directory.Delete(TestPath, recursive: true); + } + } + + static void RunAndUnload(AppDomain domain, string assemblyPath) + { + // Somehow the stack traces during execution sometimes have the wrong line numbers. + // Add some info for when debugging is required. + Console.WriteLine($"-- Running domain {domain.FriendlyName}"); + domain.ExecuteAssembly(assemblyPath); + AppDomain.Unload(domain); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } + + static string CreateTestClassAssembly(string code) + { + return CreateAssembly(TestAssemblyName + ".dll", code, exe: false); + } + + static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "ShutdownMode.Reload") + { + var code = string.Format(CaseRunnerTemplate, shutdownMode, verb); + var name = "TestCaseRunner.exe"; + + return CreateAssembly(name, code, exe: true); + } + static string CreateAssembly(string name, string code, bool exe = false) + { + // Never return or hold the Assembly instance. This will cause + // the assembly to be loaded into the current domain and this + // interferes with the tests. The Domain can execute fine from a + // path, so let's return that. + CSharpCodeProvider provider = new CSharpCodeProvider(); + CompilerParameters parameters = new CompilerParameters(); + parameters.GenerateExecutable = exe; + var assemblyName = name; + var assemblyFullPath = Path.Combine(TestPath, assemblyName); + parameters.OutputAssembly = assemblyFullPath; + parameters.ReferencedAssemblies.Add("System.dll"); + parameters.ReferencedAssemblies.Add("System.Core.dll"); + parameters.ReferencedAssemblies.Add("Microsoft.CSharp.dll"); + var netstandard = "netstandard.dll"; + if (Type.GetType("Mono.Runtime") != null) + { + netstandard = "Facades/" + netstandard; + } + parameters.ReferencedAssemblies.Add(netstandard); + parameters.ReferencedAssemblies.Add(PythonDllLocation); + CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + if (results.NativeCompilerReturnValue != 0) + { + var stderr = System.Console.Error; + stderr.WriteLine($"Error in {name} compiling:\n{code}"); + foreach (var error in results.Errors) + { + stderr.WriteLine(error); + } + throw new ArgumentException("Error compiling code"); + } + + return assemblyFullPath; + } + + static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = TestPath, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + + return domain; + } + + static string CreatePythonModule(TestCase testCase) + { + var modulePath = Path.Combine(TestPath, "domain_test_module"); + if (Directory.Exists(modulePath)) + { + Directory.Delete(modulePath, recursive: true); + } + Directory.CreateDirectory(modulePath); + + File.Create(Path.Combine(modulePath, "__init__.py")).Close(); //Create and don't forget to close! + using (var writer = File.CreateText(Path.Combine(modulePath, "mod.py"))) + { + writer.Write(testCase.PythonCode); + } + + return null; + } + } +} diff --git a/src/domain_tests/conftest.py b/src/domain_tests/conftest.py new file mode 100644 index 000000000..5f0d52e10 --- /dev/null +++ b/src/domain_tests/conftest.py @@ -0,0 +1,7 @@ +import os + +from subprocess import check_call +# test_proj_path = os.path.join(cwd, "..", "testing") +cfd = os.path.dirname(__file__) +bin_path = os.path.join(cfd, 'bin') +check_call(["dotnet", "build", cfd, '-o', bin_path]) \ No newline at end of file diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py new file mode 100644 index 000000000..e24eb6976 --- /dev/null +++ b/src/domain_tests/test_domain_reload.py @@ -0,0 +1,100 @@ +import subprocess +import os +import platform + +import pytest + +def _run_test(testname): + dirname = os.path.split(__file__)[0] + exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') + args = [exename, testname] + + if platform.system() != 'Windows': + args = ['mono'] + args + + proc = subprocess.Popen(args) + proc.wait() + + assert proc.returncode == 0 + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class(): + _run_test('class_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_static_function(): + _run_test('static_member_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_function(): + _run_test('member_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_field(): + _run_test('field_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_property(): + _run_test('property_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_namespace(): + _run_test('namespace_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_field_visibility_change(): + _run_test("field_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_method_visibility_change(): + _run_test("method_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_property_visibility_change(): + _run_test("property_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_class_visibility_change(): + _run_test("class_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_method_parameters_change(): + _run_test("method_parameters_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_method_return_type_change(): + _run_test("method_return_type_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_field_type_change(): + _run_test("field_type_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.xfail(reason="Events not yet serializable") +def test_rename_event(): + _run_test('event_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") +def test_construct_removed_class(): + _run_test("construct_removed_class") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_out_to_ref_param(): + _run_test("out_to_ref_param") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_ref_to_out_param(): + _run_test("ref_to_out_param") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_ref_to_in_param(): + _run_test("ref_to_in_param") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_in_to_ref_param(): + _run_test("in_to_ref_param") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_nested_type(): + _run_test("nested_type") diff --git a/src/embed_tests/CodecGroups.cs b/src/embed_tests/CodecGroups.cs new file mode 100644 index 000000000..5dd40210f --- /dev/null +++ b/src/embed_tests/CodecGroups.cs @@ -0,0 +1,132 @@ +namespace Python.EmbeddingTest +{ + using System; + using System.Linq; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class CodecGroups + { + [Test] + public void GetEncodersByType() + { + var encoder1 = new ObjectToEncoderInstanceEncoder(); + var encoder2 = new ObjectToEncoderInstanceEncoder(); + var group = new EncoderGroup { + new ObjectToEncoderInstanceEncoder>(), + encoder1, + encoder2, + }; + + var got = group.GetEncoders(typeof(Uri)).ToArray(); + CollectionAssert.AreEqual(new[]{encoder1, encoder2}, got); + } + + [Test] + public void CanEncode() + { + var group = new EncoderGroup { + new ObjectToEncoderInstanceEncoder>(), + new ObjectToEncoderInstanceEncoder(), + }; + + Assert.IsTrue(group.CanEncode(typeof(Tuple))); + Assert.IsTrue(group.CanEncode(typeof(Uri))); + Assert.IsFalse(group.CanEncode(typeof(string))); + } + + [Test] + public void Encodes() + { + var encoder0 = new ObjectToEncoderInstanceEncoder>(); + var encoder1 = new ObjectToEncoderInstanceEncoder(); + var encoder2 = new ObjectToEncoderInstanceEncoder(); + var group = new EncoderGroup { + encoder0, + encoder1, + encoder2, + }; + + var uri = group.TryEncode(new Uri("data:")); + var clrObject = (CLRObject)ManagedType.GetManagedObject(uri.Handle); + Assert.AreSame(encoder1, clrObject.inst); + Assert.AreNotSame(encoder2, clrObject.inst); + + var tuple = group.TryEncode(Tuple.Create(1)); + clrObject = (CLRObject)ManagedType.GetManagedObject(tuple.Handle); + Assert.AreSame(encoder0, clrObject.inst); + } + + [Test] + public void GetDecodersByTypes() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var pystr = new PyString("world").GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + var decoder = group.GetDecoder(pyfloat, typeof(string)); + Assert.AreSame(decoder2, decoder); + decoder = group.GetDecoder(pystr, typeof(string)); + Assert.IsNull(decoder); + decoder = group.GetDecoder(pyint, typeof(long)); + Assert.AreSame(decoder1, decoder); + } + [Test] + public void CanDecode() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var pystr = new PyString("world").GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + Assert.IsTrue(group.CanDecode(pyint, typeof(long))); + Assert.IsFalse(group.CanDecode(pyint, typeof(int))); + Assert.IsTrue(group.CanDecode(pyfloat, typeof(string))); + Assert.IsFalse(group.CanDecode(pystr, typeof(string))); + } + + [Test] + public void Decodes() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + Assert.IsTrue(group.TryDecode(new PyInt(10), out long longResult)); + Assert.AreEqual(42, longResult); + Assert.IsTrue(group.TryDecode(new PyFloat(10), out string strResult)); + Assert.AreSame("atad:", strResult); + + Assert.IsFalse(group.TryDecode(new PyInt(10), out int _)); + } + + [SetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + } +} diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs new file mode 100644 index 000000000..18fcd32d1 --- /dev/null +++ b/src/embed_tests/Codecs.cs @@ -0,0 +1,123 @@ +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Text; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == TheOnlySupportedSourceType.Handle + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } +} diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs new file mode 100644 index 000000000..458ab6a99 --- /dev/null +++ b/src/embed_tests/GlobalTestsSetup.cs @@ -0,0 +1,21 @@ +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + + // As the SetUpFixture, the OneTimeTearDown of this class is executed after + // all tests have run. + [SetUpFixture] + public class GlobalTestsSetup + { + [OneTimeTearDown] + public void FinalCleanup() + { + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + } + } +} diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 9c417cd27..1bb4fed11 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,119 +1,33 @@ - - + + - Debug - AnyCPU - {4165C59D-2822-499F-A6DB-EACA4C331EB5} - Library - Python.EmbeddingTest - Python.EmbeddingTest - bin\Python.EmbeddingTest.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - 6 - true - prompt + net472;netcoreapp3.1 - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - - - ..\..\packages\NUnit.3.6.0\lib\net40\nunit.framework.dll - - - - - - - + - - - - - - - - - - - - - - - - - + + - - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Python.Runtime - + - - - - + - $(TargetPath) - $(TargetDir)$(TargetName).pdb + $(DefineConstants);$(ConfiguredConstants) - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + 1.0.0 + all + runtime; build; native; contentfiles; analyzers + + + diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs new file mode 100644 index 000000000..1d29e85c7 --- /dev/null +++ b/src/embed_tests/References.cs @@ -0,0 +1,55 @@ +namespace Python.EmbeddingTest +{ + using NUnit.Framework; + using Python.Runtime; + + public class References + { + private Py.GILState _gs; + + [SetUp] + public void SetUp() + { + _gs = Py.GIL(); + } + + [TearDown] + public void Dispose() + { + _gs.Dispose(); + } + + [Test] + public void MoveToPyObject_SetsNull() + { + var dict = new PyDict(); + NewReference reference = Runtime.PyDict_Items(dict.Handle); + try + { + Assert.IsFalse(reference.IsNull()); + + using (reference.MoveToPyObject()) + Assert.IsTrue(reference.IsNull()); + } + finally + { + reference.Dispose(); + } + } + + [Test] + public void CanBorrowFromNewReference() + { + var dict = new PyDict(); + NewReference reference = Runtime.PyDict_Items(dict.Handle); + try + { + PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference)); + } + finally + { + reference.Dispose(); + } + } + } +} diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs new file mode 100644 index 000000000..454c97578 --- /dev/null +++ b/src/embed_tests/TestCallbacks.cs @@ -0,0 +1,33 @@ +using System; + +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest { + using Runtime = Python.Runtime.Runtime; + + public class TestCallbacks { + [OneTimeSetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void TestNoOverloadException() { + int passed = 0; + var aFunctionThatCallsIntoPython = new Action(value => passed = value); + using (Py.GIL()) { + dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); + var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); + Assert.AreEqual("TypeError", error.PythonTypeName); + string expectedArgTypes = "()"; + StringAssert.EndsWith(expectedArgTypes, error.Message); + } + } + } +} diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs new file mode 100644 index 000000000..875adf8ef --- /dev/null +++ b/src/embed_tests/TestConverter.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using NUnit.Framework; + +using Python.Runtime; + +using PyRuntime = Python.Runtime.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestConverter + { + static readonly Type[] _numTypes = new Type[] + { + typeof(short), + typeof(ushort), + typeof(int), + typeof(uint), + typeof(long), + typeof(ulong) + }; + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestConvertSingleToManaged( + [Values(float.PositiveInfinity, float.NegativeInfinity, float.MinValue, float.MaxValue, float.NaN, + float.Epsilon)] float testValue) + { + var pyFloat = new PyFloat(testValue); + + object convertedValue; + var converted = Converter.ToManaged(pyFloat.Handle, typeof(float), out convertedValue, false); + + Assert.IsTrue(converted); + Assert.IsTrue(((float) convertedValue).Equals(testValue)); + } + + [Test] + public void TestConvertDoubleToManaged( + [Values(double.PositiveInfinity, double.NegativeInfinity, double.MinValue, double.MaxValue, double.NaN, + double.Epsilon)] double testValue) + { + var pyFloat = new PyFloat(testValue); + + object convertedValue; + var converted = Converter.ToManaged(pyFloat.Handle, typeof(double), out convertedValue, false); + + Assert.IsTrue(converted); + Assert.IsTrue(((double) convertedValue).Equals(testValue)); + } + + [Test] + public void CovertTypeError() + { + Type[] floatTypes = new Type[] + { + typeof(float), + typeof(double) + }; + using (var s = new PyString("abc")) + { + foreach (var type in _numTypes.Union(floatTypes)) + { + object value; + try + { + bool res = Converter.ToManaged(s.Handle, type, out value, true); + Assert.IsFalse(res); + var bo = Exceptions.ExceptionMatches(Exceptions.TypeError); + Assert.IsTrue(Exceptions.ExceptionMatches(Exceptions.TypeError) + || Exceptions.ExceptionMatches(Exceptions.ValueError)); + } + finally + { + Exceptions.Clear(); + } + } + } + } + + [Test] + public void ConvertOverflow() + { + using (var num = new PyLong(ulong.MaxValue)) + { + IntPtr largeNum = PyRuntime.PyNumber_Add(num.Handle, num.Handle); + try + { + object value; + foreach (var type in _numTypes) + { + bool res = Converter.ToManaged(largeNum, type, out value, true); + Assert.IsFalse(res); + Assert.IsTrue(Exceptions.ExceptionMatches(Exceptions.OverflowError)); + Exceptions.Clear(); + } + } + finally + { + Exceptions.Clear(); + PyRuntime.XDecref(largeNum); + } + } + } + + [Test] + public void RawListProxy() + { + var list = new List {"hello", "world"}; + var listProxy = PyObject.FromManagedObject(list); + var clrObject = (CLRObject)ManagedType.GetManagedObject(listProxy.Handle); + Assert.AreSame(list, clrObject.inst); + } + + [Test] + public void RawPyObjectProxy() + { + var pyObject = "hello world!".ToPython(); + var pyObjectProxy = PyObject.FromManagedObject(pyObject); + var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy.Handle); + Assert.AreSame(pyObject, clrObject.inst); + + var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); + Assert.AreEqual(pyObject.Handle, proxiedHandle); + } + } +} diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs new file mode 100644 index 000000000..f8445edb4 --- /dev/null +++ b/src/embed_tests/TestDomainReload.cs @@ -0,0 +1,530 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using NUnit.Framework; +using Python.Runtime; + +using PyRuntime = Python.Runtime.Runtime; +// +// This test case is disabled on .NET Standard because it doesn't have all the +// APIs we use. We could work around that, but .NET Core doesn't implement +// domain creation, so it's not worth it. +// +// Unfortunately this means no continuous integration testing for this case. +// +#if NETFRAMEWORK +namespace Python.EmbeddingTest +{ + class TestDomainReload + { + abstract class CrossCaller : MarshalByRefObject + { + public abstract ValueType Execute(ValueType arg); + } + + + /// + /// Test that the python runtime can survive a C# domain reload without crashing. + /// + /// At the time this test was written, there was a very annoying + /// seemingly random crash bug when integrating pythonnet into Unity. + /// + /// The repro steps that David Lassonde, Viktoria Kovecses and + /// Benoit Hudson eventually worked out: + /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access + /// some C# data from python: C# calls python, which calls C#. + /// 2. Execute the script (e.g. make it a MenuItem and click it). + /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. + /// 4. Wait several seconds for Unity to be done recompiling and + /// reloading the C# domain. + /// 5. Make python run the gc (e.g. by calling gc.collect()). + /// + /// The reason: + /// A. In step 2, Python.Runtime registers a bunch of new types with + /// their tp_traverse slot pointing to managed code, and allocates + /// some objects of those types. + /// B. In step 4, Unity unloads the C# domain. That frees the managed + /// code. But at the time of the crash investigation, pythonnet + /// leaked the python side of the objects allocated in step 1. + /// C. In step 5, python sees some pythonnet objects in its gc list of + /// potentially-leaked objects. It calls tp_traverse on those objects. + /// But tp_traverse was freed in step 3 => CRASH. + /// + /// This test distills what's going on without needing Unity around (we'd see + /// similar behaviour if we were using pythonnet on a .NET web server that did + /// a hot reload). + /// + [Test] + public static void DomainReloadAndGC() + { + Assert.IsFalse(PythonEngine.IsInitialized); + RunAssemblyAndUnload("test1"); + Assert.That(PyRuntime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + RunAssemblyAndUnload("test2"); + Assert.That(PyRuntime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + // The default mode is a normal mode, + // it should shutdown the Python VM avoiding influence other tests. + PyRuntime.PyGILState_Ensure(); + PyRuntime.Py_Finalize(); + } + } + + #region CrossDomainObject + + class CrossDomainObjectStep1 : CrossCaller + { + public override ValueType Execute(ValueType arg) + { + try + { + // Create a C# user-defined object in Python. Asssing some values. + Type type = typeof(Python.EmbeddingTest.Domain.MyClass); + string code = string.Format(@" +import clr +clr.AddReference('{0}') + +from Python.EmbeddingTest.Domain import MyClass +obj = MyClass() +obj.Method() +obj.StaticMethod() +obj.Property = 1 +obj.Field = 10 +", Assembly.GetExecutingAssembly().FullName); + + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(code); + using (PyObject obj = scope.Get("obj")) + { + Debug.Assert(obj.AsManagedObject(type).GetType() == type); + // We only needs its Python handle + PyRuntime.XIncref(obj.Handle); + return obj.Handle; + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + } + } + + + class CrossDomainObjectStep2 : CrossCaller + { + public override ValueType Execute(ValueType arg) + { + // handle refering a clr object created in previous domain, + // it should had been deserialized and became callable agian. + IntPtr handle = (IntPtr)arg; + try + { + using (Py.GIL()) + { + IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); + IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); + Assert.That(tp_clear, Is.Not.Null); + + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + + using (var scope = Py.CreateScope()) + { + scope.Set("obj", obj); + scope.Exec(@" +obj.Method() +obj.StaticMethod() +obj.Property += 1 +obj.Field += 10 +"); + } + var clrObj = obj.As(); + Assert.AreEqual(clrObj.Property, 2); + Assert.AreEqual(clrObj.Field, 20); + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + return 0; + } + } + + /// + /// Create a C# custom object in a domain, in python code. + /// Unload the domain, create a new domain. + /// Make sure the C# custom object created in the previous domain has been re-created + /// + [Test] + public static void CrossDomainObject() + { + RunDomainReloadSteps(); + } + + #endregion + + #region Tempary tests + + // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 + [Test] + public void CrossReleaseBuiltinType() + { + void ExecTest() + { + try + { + PythonEngine.Initialize(); + var numRef = CreateNumReference(); + Assert.True(numRef.IsAlive); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + + GC.Collect(); + GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue + Finalizer.Instance.Collect(); + // ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`, + // but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead. + Assert.False(numRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) + { + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; + try + { + for (int i = 0; i < 10; i++) + { + ExecTest(); + } + } + finally + { + Finalizer.Instance.ErrorHandler -= ErrorHandler; + } + Assert.AreEqual(errorArgs.Count, 0); + } + + [Test] + public void CrossReleaseCustomType() + { + void ExecTest() + { + try + { + PythonEngine.Initialize(); + var objRef = CreateConcreateObject(); + Assert.True(objRef.IsAlive); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + GC.Collect(); + GC.WaitForPendingFinalizers(); + Finalizer.Instance.Collect(); + Assert.False(objRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) + { + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; + try + { + for (int i = 0; i < 10; i++) + { + ExecTest(); + } + } + finally + { + Finalizer.Instance.ErrorHandler -= ErrorHandler; + } + Assert.AreEqual(errorArgs.Count, 0); + } + + private static WeakReference CreateNumReference() + { + var num = 3216757418.ToPython(); + Assert.AreEqual(num.Refcount, 1); + WeakReference numRef = new WeakReference(num, false); + return numRef; + } + + private static WeakReference CreateConcreateObject() + { + var obj = new Domain.MyClass().ToPython(); + Assert.AreEqual(obj.Refcount, 1); + WeakReference numRef = new WeakReference(obj, false); + return numRef; + } + + #endregion Tempary tests + + /// + /// This is a magic incantation required to run code in an application + /// domain other than the current one. + /// + class Proxy : MarshalByRefObject + { + public void RunPython() + { + Console.WriteLine("[Proxy] Entering RunPython"); + PythonRunner.RunPython(); + Console.WriteLine("[Proxy] Leaving RunPython"); + } + + public object Call(string methodName, params object[] args) + { + var pythonrunner = typeof(PythonRunner); + var method = pythonrunner.GetMethod(methodName); + return method.Invoke(null, args); + } + } + + static T CreateInstanceInstanceAndUnwrap(AppDomain domain) + { + Type type = typeof(T); + var theProxy = (T)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + return theProxy; + } + + /// + /// Create a domain, run the assembly in it (the RunPython function), + /// and unload the domain. + /// + static void RunAssemblyAndUnload(string domainName) + { + Console.WriteLine($"[Program.Main] === creating domain {domainName}"); + + AppDomain domain = CreateDomain(domainName); + // Create a Proxy object in the new domain, where we want the + // assembly (and Python .NET) to reside + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + + theProxy.Call("InitPython", ShutdownMode.Soft); + // From now on use the Proxy to call into the new assembly + theProxy.RunPython(); + + theProxy.Call("ShutdownPython"); + Console.WriteLine($"[Program.Main] Before Domain Unload on {domainName}"); + AppDomain.Unload(domain); + Console.WriteLine($"[Program.Main] After Domain Unload on {domainName}"); + + // Validate that the assembly does not exist anymore + try + { + Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); + Assert.Fail($"{theProxy} should be invlaid now"); + } + catch (AppDomainUnloadedException) + { + Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); + } + } + + private static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = currentDomain.SetupInformation.ApplicationBase, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + return domain; + } + + /// + /// Resolves the assembly. Why doesn't this just work normally? + /// + static Assembly ResolveAssembly(object sender, ResolveEventArgs args) + { + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + + foreach (var assembly in loadedAssemblies) + { + if (assembly.FullName == args.Name) + { + return assembly; + } + } + + return null; + } + + static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : CrossCaller + { + ValueType arg = null; + Type type = typeof(Proxy); + { + AppDomain domain = CreateDomain("test_domain_reload_1"); + try + { + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = CreateInstanceInstanceAndUnwrap(domain); + arg = caller.Execute(arg); + + theProxy.Call("ShutdownPython"); + } + finally + { + AppDomain.Unload(domain); + } + } + + { + AppDomain domain = CreateDomain("test_domain_reload_2"); + try + { + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = CreateInstanceInstanceAndUnwrap(domain); + caller.Execute(arg); + theProxy.Call("ShutdownPythonCompletely"); + } + finally + { + AppDomain.Unload(domain); + } + } + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0); + } + } + } + + + // + // The code we'll test. All that really matters is + // using GIL { Python.Exec(pyScript); } + // but the rest is useful for debugging. + // + // What matters in the python code is gc.collect and clr.AddReference. + // + // Note that the language version is 2.0, so no $"foo{bar}" syntax. + // + static class PythonRunner + { + public static void RunPython() + { + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + string name = AppDomain.CurrentDomain.FriendlyName; + Console.WriteLine("[{0} in .NET] In PythonRunner.RunPython", name); + using (Py.GIL()) + { + try + { + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + throw; + } + } + } + + + private static IntPtr _state; + + public static void InitPython(ShutdownMode mode) + { + PythonEngine.Initialize(mode: mode); + _state = PythonEngine.BeginAllowThreads(); + } + + public static void ShutdownPython() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.Shutdown(); + } + + public static void ShutdownPythonCompletely() + { + PythonEngine.EndAllowThreads(_state); + // XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`, + // if it used a another mode(the default mode) in other tests, + // when other tests trying to access these reserved objects, it may cause Domain exception, + // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python. + var defaultMode = PythonEngine.DefaultShutdownMode; + if (defaultMode != ShutdownMode.Reload) + { + PythonEngine.ShutdownMode = defaultMode; + } + PythonEngine.Shutdown(); + } + + static void OnDomainUnload(object sender, EventArgs e) + { + Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); + } + } + +} + + +namespace Python.EmbeddingTest.Domain +{ + [Serializable] + public class MyClass + { + public int Property { get; set; } + public int Field; + public void Method() { } + public static void StaticMethod() { } + } +} + + +#endif diff --git a/src/embed_tests/TestExample.cs b/src/embed_tests/TestExample.cs new file mode 100644 index 000000000..671f9e33d --- /dev/null +++ b/src/embed_tests/TestExample.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestExample + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestReadme() + { + dynamic np; + try + { + np = Py.Import("numpy"); + } + catch (PythonException) + { + Assert.Inconclusive("Numpy or dependency not installed"); + return; + } + + Assert.AreEqual("1.0", np.cos(np.pi * 2).ToString()); + + dynamic sin = np.sin; + StringAssert.StartsWith("-0.95892", sin(5).ToString()); + + double c = np.cos(5) + sin(5); + Assert.AreEqual(-0.675262, c, 0.01); + + dynamic a = np.array(new List { 1, 2, 3 }); + Assert.AreEqual("float64", a.dtype.ToString()); + + dynamic b = np.array(new List { 6, 5, 4 }, Py.kw("dtype", np.int32)); + Assert.AreEqual("int32", b.dtype.ToString()); + + Assert.AreEqual("[ 6. 10. 12.]", (a * b).ToString().Replace(" ", " ")); + } + } +} diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..46e2fcdf1 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,231 @@ +using NUnit.Framework; +using Python.Runtime; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + private int _oldThreshold; + + [SetUp] + public void SetUp() + { + _oldThreshold = Finalizer.Instance.Threshold; + PythonEngine.Initialize(); + Exceptions.Clear(); + } + + [TearDown] + public void TearDown() + { + Finalizer.Instance.Threshold = _oldThreshold; + PythonEngine.Shutdown(); + } + + private static void FullGCCollect() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + [Test] + [Obsolete("GC tests are not guaranteed")] + public void CollectBasicObject() + { + Assert.IsTrue(Finalizer.Instance.Enable); + + Finalizer.Instance.Threshold = 1; + bool called = false; + var objectCount = 0; + EventHandler handler = (s, e) => + { + objectCount = e.ObjectCount; + called = true; + }; + + Assert.IsFalse(called, "The event handler was called before it was installed"); + Finalizer.Instance.CollectOnce += handler; + + IntPtr pyObj = MakeAGarbage(out var shortWeak, out var longWeak); + FullGCCollect(); + // The object has been resurrected + Warn.If( + shortWeak.IsAlive, + "The referenced object is alive although it should have been collected", + shortWeak + ); + Assert.IsTrue( + longWeak.IsAlive, + "The reference object is not alive although it should still be", + longWeak + ); + + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count, "There should still be garbage around"); + Warn.Unless( + garbage.Contains(pyObj), + $"The {nameof(longWeak)} reference doesn't show up in the garbage list", + garbage + ); + } + try + { + Finalizer.Instance.Collect(); + } + finally + { + Finalizer.Instance.CollectOnce -= handler; + } + Assert.IsTrue(called, "The event handler was not called during finalization"); + Assert.GreaterOrEqual(objectCount, 1); + } + + [Test] + [Obsolete("GC tests are not guaranteed")] + public void CollectOnShutdown() + { + IntPtr op = MakeAGarbage(out var shortWeak, out var longWeak); + FullGCCollect(); + Assert.IsFalse(shortWeak.IsAlive); + List garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.IsNotEmpty(garbage, "The garbage object should be collected"); + Assert.IsTrue(garbage.Contains(op), + "Garbage should contains the collected object"); + + PythonEngine.Shutdown(); + garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.IsEmpty(garbage); + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] // ensure lack of references to obj + [Obsolete("GC tests are not guaranteed")] + private static IntPtr MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + { + IntPtr handle = IntPtr.Zero; + WeakReference @short = null, @long = null; + // must create Python object in the thread where we have GIL + IntPtr val = PyLong.FromLong(1024); + // must create temp object in a different thread to ensure it is not present + // when conservatively scanning stack for GC roots. + // see https://xamarin.github.io/bugzilla-archives/17/17593/bug.html + var garbageGen = new Thread(() => + { + var obj = new PyObject(val, skipCollect: true); + @short = new WeakReference(obj); + @long = new WeakReference(obj, true); + handle = obj.Handle; + }); + garbageGen.Start(); + Assert.IsTrue(garbageGen.Join(TimeSpan.FromSeconds(5)), "Garbage creation timed out"); + shortWeak = @short; + longWeak = @long; + return handle; + } + + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) + { + // Must larger than 512 bytes make sure Python use + string str = new string('1', 1024); + Finalizer.Instance.Enable = true; + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + Finalizer.Instance.Collect(); + Finalizer.Instance.Enable = enbale; + + // Estimate unmanaged memory size + long before = Environment.WorkingSet - GC.GetTotalMemory(true); + for (int i = 0; i < 10000; i++) + { + // Memory will leak when disable Finalizer + new PyString(str); + } + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + if (enbale) + { + Finalizer.Instance.Collect(); + } + + FullGCCollect(); + FullGCCollect(); + long after = Environment.WorkingSet - GC.GetTotalMemory(true); + return after - before; + + } + + /// + /// Because of two vms both have their memory manager, + /// this test only prove the finalizer has take effect. + /// + [Test] + [Ignore("Too many uncertainties, only manual on when debugging")] + public void SimpleTestMemory() + { + bool oldState = Finalizer.Instance.Enable; + try + { + using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyObject pyCollect = gcModule.GetAttr("collect")) + { + long span1 = CompareWithFinalizerOn(pyCollect, false); + long span2 = CompareWithFinalizerOn(pyCollect, true); + Assert.Less(span2, span1); + } + } + finally + { + Finalizer.Instance.Enable = oldState; + } + } + + [Test] + public void ValidateRefCount() + { + if (!Finalizer.Instance.RefCountValidationEnabled) + { + Assert.Ignore("Only run with FINALIZER_CHECK"); + } + IntPtr ptr = IntPtr.Zero; + bool called = false; + Finalizer.IncorrectRefCntHandler handler = (s, e) => + { + called = true; + Assert.AreEqual(ptr, e.Handle); + Assert.AreEqual(2, e.ImpactedObjects.Count); + // Fix for this test, don't do this on general environment + Runtime.Runtime.XIncref(e.Handle); + return false; + }; + Finalizer.Instance.IncorrectRefCntResolver += handler; + try + { + ptr = CreateStringGarbage(); + FullGCCollect(); + Assert.Throws(() => Finalizer.Instance.Collect()); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.IncorrectRefCntResolver -= handler; + } + } + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] // ensure lack of references to s1 and s2 + private static IntPtr CreateStringGarbage() + { + PyString s1 = new PyString("test_string"); + // s2 steal a reference from s1 + PyString s2 = new PyString(s1.Handle); + return s1.Handle; + } + } +} diff --git a/src/embed_tests/TestGILState.cs b/src/embed_tests/TestGILState.cs new file mode 100644 index 000000000..bf6f02dc6 --- /dev/null +++ b/src/embed_tests/TestGILState.cs @@ -0,0 +1,33 @@ +namespace Python.EmbeddingTest +{ + using NUnit.Framework; + using Python.Runtime; + + public class TestGILState + { + /// + /// Ensure, that calling multiple times is safe + /// + [Test] + public void CanDisposeMultipleTimes() + { + using (var gilState = Py.GIL()) + { + for(int i = 0; i < 50; i++) + gilState.Dispose(); + } + } + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + } +} diff --git a/src/embed_tests/TestInstanceWrapping.cs b/src/embed_tests/TestInstanceWrapping.cs new file mode 100644 index 000000000..8be207c00 --- /dev/null +++ b/src/embed_tests/TestInstanceWrapping.cs @@ -0,0 +1,58 @@ +using System; +using System.Globalization; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestInstanceWrapping + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + // regression test for https://github.com/pythonnet/pythonnet/issues/811 + [Test] + public void OverloadResolution_UnknownToObject() + { + var overloaded = new Overloaded(); + using (Py.GIL()) + { + var o = overloaded.ToPython(); + + dynamic callWithSelf = PythonEngine.Eval("lambda o: o.ObjOrClass(object())"); + callWithSelf(o); + Assert.AreEqual(Overloaded.Object, overloaded.Value); + } + } + + class Base {} + class Derived: Base { } + + class Overloaded: Derived + { + public int Value { get; set; } + public void IntOrStr(int arg) => this.Value = arg; + public void IntOrStr(string arg) => + this.Value = int.Parse(arg, NumberStyles.Integer, CultureInfo.InvariantCulture); + + public const int Object = 1; + public const int ConcreteClass = 2; + public void ObjOrClass(object _) => this.Value = Object; + public void ObjOrClass(Overloaded _) => this.Value = ConcreteClass; + + public const int Base = ConcreteClass + 1; + public const int Derived = Base + 1; + public void BaseOrDerived(Base _) => this.Value = Base; + public void BaseOrDerived(Derived _) => this.Value = Derived; + } + } +} diff --git a/src/embed_tests/TestNamedArguments.cs b/src/embed_tests/TestNamedArguments.cs new file mode 100644 index 000000000..31f2ea1d2 --- /dev/null +++ b/src/embed_tests/TestNamedArguments.cs @@ -0,0 +1,64 @@ +using System; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestNamedArguments + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + /// + /// Test named arguments support through Py.kw method + /// + [Test] + public void TestKeywordArgs() + { + dynamic a = CreateTestClass(); + var result = (int)a.Test3(2, Py.kw("a4", 8)); + + Assert.AreEqual(12, result); + } + + + /// + /// Test keyword arguments with .net named arguments + /// + [Test] + public void TestNamedArgs() + { + dynamic a = CreateTestClass(); + var result = (int)a.Test3(2, a4: 8); + + Assert.AreEqual(12, result); + } + + + + private static PyObject CreateTestClass() + { + var locals = new PyDict(); + + PythonEngine.Exec(@" +class cmTest3: + def Test3(self, a1 = 1, a2 = 1, a3 = 1, a4 = 1): + return a1 + a2 + a3 + a4 + +a = cmTest3() +", null, locals.Handle); + + return locals.GetItem("a"); + } + + } +} diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs new file mode 100644 index 000000000..03812c6fe --- /dev/null +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingPythonTest +{ + public class TestNativeTypeOffset + { + private Py.GILState _gs; + + [SetUp] + public void SetUp() + { + _gs = Py.GIL(); + } + + [TearDown] + public void Dispose() + { + _gs.Dispose(); + } + + /// + /// Tests that installation has generated code for NativeTypeOffset and that it can be loaded. + /// + [Test] + public void LoadNativeTypeOffsetClass() + { + PyObject sys = Py.Import("sys"); + string attributeName = "abiflags"; + if (sys.HasAttr(attributeName) && !string.IsNullOrEmpty(sys.GetAttr(attributeName).ToString())) + { + string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; + Assert.NotNull(Type.GetType(typeName), $"{typeName} does not exist and sys.{attributeName} is not empty"); + } + } + } +} diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs new file mode 100644 index 000000000..8e9feb241 --- /dev/null +++ b/src/embed_tests/TestOperator.cs @@ -0,0 +1,521 @@ +using NUnit.Framework; + +using Python.Runtime; + +using System.Linq; +using System.Reflection; + +namespace Python.EmbeddingTest +{ + public class TestOperator + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + public class OperableObject + { + public int Num { get; set; } + + public override int GetHashCode() + { + return unchecked(159832395 + Num.GetHashCode()); + } + + public override bool Equals(object obj) + { + return obj is OperableObject @object && + Num == @object.Num; + } + + public OperableObject(int num) + { + Num = num; + } + + public static OperableObject operator ~(OperableObject a) + { + return new OperableObject(~a.Num); + } + + public static OperableObject operator +(OperableObject a) + { + return new OperableObject(+a.Num); + } + + public static OperableObject operator -(OperableObject a) + { + return new OperableObject(-a.Num); + } + + public static OperableObject operator +(int a, OperableObject b) + { + return new OperableObject(a + b.Num); + } + public static OperableObject operator +(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num + b.Num); + } + public static OperableObject operator +(OperableObject a, int b) + { + return new OperableObject(a.Num + b); + } + + public static OperableObject operator -(int a, OperableObject b) + { + return new OperableObject(a - b.Num); + } + public static OperableObject operator -(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num - b.Num); + } + public static OperableObject operator -(OperableObject a, int b) + { + return new OperableObject(a.Num - b); + } + + public static OperableObject operator *(int a, OperableObject b) + { + return new OperableObject(a * b.Num); + } + public static OperableObject operator *(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num * b.Num); + } + public static OperableObject operator *(OperableObject a, int b) + { + return new OperableObject(a.Num * b); + } + + public static OperableObject operator /(int a, OperableObject b) + { + return new OperableObject(a / b.Num); + } + public static OperableObject operator /(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num / b.Num); + } + public static OperableObject operator /(OperableObject a, int b) + { + return new OperableObject(a.Num / b); + } + + public static OperableObject operator %(int a, OperableObject b) + { + return new OperableObject(a % b.Num); + } + public static OperableObject operator %(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num % b.Num); + } + public static OperableObject operator %(OperableObject a, int b) + { + return new OperableObject(a.Num % b); + } + + public static OperableObject operator &(int a, OperableObject b) + { + return new OperableObject(a & b.Num); + } + public static OperableObject operator &(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num & b.Num); + } + public static OperableObject operator &(OperableObject a, int b) + { + return new OperableObject(a.Num & b); + } + + public static OperableObject operator |(int a, OperableObject b) + { + return new OperableObject(a | b.Num); + } + public static OperableObject operator |(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num | b.Num); + } + public static OperableObject operator |(OperableObject a, int b) + { + return new OperableObject(a.Num | b); + } + + public static OperableObject operator ^(int a, OperableObject b) + { + return new OperableObject(a ^ b.Num); + } + public static OperableObject operator ^(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num ^ b.Num); + } + public static OperableObject operator ^(OperableObject a, int b) + { + return new OperableObject(a.Num ^ b); + } + + public static bool operator ==(int a, OperableObject b) + { + return (a == b.Num); + } + public static bool operator ==(OperableObject a, OperableObject b) + { + return (a.Num == b.Num); + } + public static bool operator ==(OperableObject a, int b) + { + return (a.Num == b); + } + + public static bool operator !=(int a, OperableObject b) + { + return (a != b.Num); + } + public static bool operator !=(OperableObject a, OperableObject b) + { + return (a.Num != b.Num); + } + public static bool operator !=(OperableObject a, int b) + { + return (a.Num != b); + } + + public static bool operator <=(int a, OperableObject b) + { + return (a <= b.Num); + } + public static bool operator <=(OperableObject a, OperableObject b) + { + return (a.Num <= b.Num); + } + public static bool operator <=(OperableObject a, int b) + { + return (a.Num <= b); + } + + public static bool operator >=(int a, OperableObject b) + { + return (a >= b.Num); + } + public static bool operator >=(OperableObject a, OperableObject b) + { + return (a.Num >= b.Num); + } + public static bool operator >=(OperableObject a, int b) + { + return (a.Num >= b); + } + + public static bool operator >=(OperableObject a, PyObject b) + { + using (Py.GIL()) + { + // Assuming b is a tuple, take the first element. + int bNum = b[0].As(); + return a.Num >= bNum; + } + } + public static bool operator <=(OperableObject a, PyObject b) + { + using (Py.GIL()) + { + // Assuming b is a tuple, take the first element. + int bNum = b[0].As(); + return a.Num <= bNum; + } + } + + public static bool operator <(int a, OperableObject b) + { + return (a < b.Num); + } + public static bool operator <(OperableObject a, OperableObject b) + { + return (a.Num < b.Num); + } + public static bool operator <(OperableObject a, int b) + { + return (a.Num < b); + } + + public static bool operator >(int a, OperableObject b) + { + return (a > b.Num); + } + public static bool operator >(OperableObject a, OperableObject b) + { + return (a.Num > b.Num); + } + public static bool operator >(OperableObject a, int b) + { + return (a.Num > b); + } + + public static OperableObject operator <<(OperableObject a, int offset) + { + return new OperableObject(a.Num << offset); + } + + public static OperableObject operator >>(OperableObject a, int offset) + { + return new OperableObject(a.Num >> offset); + } + } + + [Test] + public void SymmetricalOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(-2) +b = cls(10) +c = ~a +assert c.Num == ~a.Num + +c = +a +assert c.Num == +a.Num + +a = cls(2) +c = -a +assert c.Num == -a.Num + +c = a + b +assert c.Num == a.Num + b.Num + +c = a - b +assert c.Num == a.Num - b.Num + +c = a * b +assert c.Num == a.Num * b.Num + +c = a / b +assert c.Num == a.Num // b.Num + +c = a % b +assert c.Num == a.Num % b.Num + +c = a & b +assert c.Num == a.Num & b.Num + +c = a | b +assert c.Num == a.Num | b.Num + +c = a ^ b +assert c.Num == a.Num ^ b.Num + +c = a == b +assert c == (a.Num == b.Num) + +c = a != b +assert c == (a.Num != b.Num) + +c = a <= b +assert c == (a.Num <= b.Num) + +c = a >= b +assert c == (a.Num >= b.Num) + +c = a < b +assert c == (a.Num < b.Num) + +c = a > b +assert c == (a.Num > b.Num) +"); + } + + [Test] + public void OperatorOverloadMissingArgument() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + Assert.Throws(() => + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = cls(10) +a.op_Addition() +")); + } + + [Test] + public void ForwardOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = 10 +c = a + b +assert c.Num == a.Num + b + +c = a - b +assert c.Num == a.Num - b + +c = a * b +assert c.Num == a.Num * b + +c = a / b +assert c.Num == a.Num // b + +c = a % b +assert c.Num == a.Num % b + +c = a & b +assert c.Num == a.Num & b + +c = a | b +assert c.Num == a.Num | b + +c = a ^ b +assert c.Num == a.Num ^ b + +c = a == b +assert c == (a.Num == b) + +c = a != b +assert c == (a.Num != b) + +c = a <= b +assert c == (a.Num <= b) + +c = a >= b +assert c == (a.Num >= b) + +c = a < b +assert c == (a.Num < b) + +c = a > b +assert c == (a.Num > b) +"); + } + + [Test] + public void TupleComparisonOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = (1, 2) + +c = a >= b +assert c == (a.Num >= b[0]) + +c = a <= b +assert c == (a.Num <= b[0]) + +c = b >= a +assert c == (b[0] >= a.Num) + +c = b <= a +assert c == (b[0] <= a.Num) +"); + } + + + [Test] + public void ReverseOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = 2 +b = cls(10) + +c = a + b +assert c.Num == a + b.Num + +c = a - b +assert c.Num == a - b.Num + +c = a * b +assert c.Num == a * b.Num + +c = a / b +assert c.Num == a // b.Num + +c = a % b +assert c.Num == a % b.Num + +c = a & b +assert c.Num == a & b.Num + +c = a | b +assert c.Num == a | b.Num + +c = a ^ b +assert c.Num == a ^ b.Num + +c = a == b +assert c == (a == b.Num) + +c = a != b +assert c == (a != b.Num) + +c = a <= b +assert c == (a <= b.Num) + +c = a >= b +assert c == (a >= b.Num) + +c = a < b +assert c == (a < b.Num) + +c = a > b +assert c == (a > b.Num) +"); + + } + [Test] + public void ShiftOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = cls(10) + +c = a << b.Num +assert c.Num == a.Num << b.Num + +c = a >> b.Num +assert c.Num == a.Num >> b.Num +"); + } + } +} diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs new file mode 100644 index 000000000..0338a1480 --- /dev/null +++ b/src/embed_tests/TestPyBuffer.cs @@ -0,0 +1,68 @@ +using System.Text; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest { + class TestPyBuffer + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestBufferWrite() + { + string bufferTestString = "hello world! !$%&/()=?"; + + using (Py.GIL()) + { + using (var scope = Py.CreateScope()) + { + scope.Exec($"arr = bytearray({bufferTestString.Length})"); + PyObject pythonArray = scope.Get("arr"); + byte[] managedArray = new UTF8Encoding().GetBytes(bufferTestString); + + using (PyBuffer buf = pythonArray.GetBuffer()) + { + buf.Write(managedArray, 0, managedArray.Length); + } + + string result = scope.Eval("arr.decode('utf-8')").ToString(); + Assert.IsTrue(result == bufferTestString); + } + } + } + + [Test] + public void TestBufferRead() + { + string bufferTestString = "hello world! !$%&/()=?"; + + using (Py.GIL()) + { + using (var scope = Py.CreateScope()) + { + scope.Exec($"arr = b'{bufferTestString}'"); + PyObject pythonArray = scope.Get("arr"); + byte[] managedArray = new byte[bufferTestString.Length]; + + using (PyBuffer buf = pythonArray.GetBuffer()) + { + buf.Read(managedArray, 0, managedArray.Length); + } + + string result = new UTF8Encoding().GetString(managedArray); + Assert.IsTrue(result == bufferTestString); + } + } + } + } +} diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index f2c85a77f..94e7026c7 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -25,6 +25,7 @@ public void Dispose() public void IntPtrCtor() { var i = new PyFloat(1); + Runtime.Runtime.XIncref(i.Handle); var ii = new PyFloat(i.Handle); Assert.AreEqual(i.Handle, ii.Handle); } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 4117336d8..005ab466d 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -86,6 +86,7 @@ public void TestCtorSByte() public void TestCtorPtr() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -94,6 +95,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyIter.cs b/src/embed_tests/TestPyIter.cs new file mode 100644 index 000000000..7428da979 --- /dev/null +++ b/src/embed_tests/TestPyIter.cs @@ -0,0 +1,37 @@ +using System.Linq; +using System.Text; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + class TestPyIter + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void KeepOldObjects() + { + using (Py.GIL()) + using (var testString = new PyString("hello world! !$%&/()=?")) + { + PyObject[] chars = testString.ToArray(); + Assert.IsTrue(chars.Length > 1); + string reconstructed = string.Concat(chars.Select(c => c.As())); + Assert.AreEqual(testString.As(), reconstructed); + } + } + } +} diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..3c155f315 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -110,6 +111,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs new file mode 100644 index 000000000..d0d8eab45 --- /dev/null +++ b/src/embed_tests/TestPyObject.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestPyObject + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestGetDynamicMemberNames() + { + List expectedMemberNames = new List + { + "add", + "getNumber", + "member1", + "member2" + }; + + PyDict locals = new PyDict(); + + PythonEngine.Exec(@" +class MemberNamesTest(object): + def __init__(self): + self.member1 = 123 + self.member2 = 'Test string' + + def getNumber(self): + return 123 + + def add(self, x, y): + return x + y + +a = MemberNamesTest() +", null, locals.Handle); + + PyObject a = locals.GetItem("a"); + + IEnumerable memberNames = a.GetDynamicMemberNames(); + + foreach (string expectedName in expectedMemberNames) + { + Assert.IsTrue(memberNames.Contains(expectedName), "Could not find member '{0}'.", expectedName); + } + } + + [Test] + public void InvokeNull() + { + var list = PythonEngine.Eval("list"); + Assert.Throws(() => list.Invoke(new PyObject[] {null})); + } + + [Test] + public void AsManagedObjectInvalidCast() + { + var list = PythonEngine.Eval("list"); + Assert.Throws(() => list.AsManagedObject(typeof(int))); + } + } +} diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs new file mode 100644 index 000000000..a94b8ce28 --- /dev/null +++ b/src/embed_tests/TestPyScope.cs @@ -0,0 +1,385 @@ +using System; +using System.Threading; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class PyScopeTest + { + private PyScope ps; + + [SetUp] + public void SetUp() + { + using (Py.GIL()) + { + ps = Py.CreateScope("test"); + } + } + + [TearDown] + public void Dispose() + { + using (Py.GIL()) + { + ps.Dispose(); + ps = null; + } + } + + /// + /// Eval a Python expression and obtain its return value. + /// + [Test] + public void TestEval() + { + using (Py.GIL()) + { + ps.Set("a", 1); + var result = ps.Eval("a + 2"); + Assert.AreEqual(3, result); + } + } + + /// + /// Exec Python statements and obtain the variables created. + /// + [Test] + public void TestExec() + { + using (Py.GIL()) + { + ps.Set("bb", 100); //declare a global variable + ps.Set("cc", 10); //declare a local variable + ps.Exec("aa = bb + cc + 3"); + var result = ps.Get("aa"); + Assert.AreEqual(113, result); + } + } + + /// + /// Compile an expression into an ast object; + /// Execute the ast and obtain its return value. + /// + [Test] + public void TestCompileExpression() + { + using (Py.GIL()) + { + ps.Set("bb", 100); //declare a global variable + ps.Set("cc", 10); //declare a local variable + PyObject script = PythonEngine.Compile("bb + cc + 3", "", RunFlagType.Eval); + var result = ps.Execute(script); + Assert.AreEqual(113, result); + } + } + + /// + /// Compile Python statements into an ast object; + /// Execute the ast; + /// Obtain the local variables created. + /// + [Test] + public void TestCompileStatements() + { + using (Py.GIL()) + { + ps.Set("bb", 100); //declare a global variable + ps.Set("cc", 10); //declare a local variable + PyObject script = PythonEngine.Compile("aa = bb + cc + 3", "", RunFlagType.File); + ps.Execute(script); + var result = ps.Get("aa"); + Assert.AreEqual(113, result); + } + } + + /// + /// Create a function in the scope, then the function can read variables in the scope. + /// It cannot write the variables unless it uses the 'global' keyword. + /// + [Test] + public void TestScopeFunction() + { + using (Py.GIL()) + { + ps.Set("bb", 100); + ps.Set("cc", 10); + ps.Exec( + "def func1():\n" + + " bb = cc + 10\n"); + dynamic func1 = ps.Get("func1"); + func1(); //call the function, it can be called any times + var result = ps.Get("bb"); + Assert.AreEqual(100, result); + + ps.Set("bb", 100); + ps.Set("cc", 10); + ps.Exec( + "def func2():\n" + + " global bb\n" + + " bb = cc + 10\n"); + dynamic func2 = ps.Get("func2"); + func2(); + result = ps.Get("bb"); + Assert.AreEqual(20, result); + } + } + + /// + /// Create a class in the scope, the class can read variables in the scope. + /// Its methods can write the variables with the help of 'global' keyword. + /// + [Test] + public void TestScopeClass() + { + using (Py.GIL()) + { + dynamic _ps = ps; + _ps.bb = 100; + ps.Exec( + "class Class1():\n" + + " def __init__(self, value):\n" + + " self.value = value\n" + + " def call(self, arg):\n" + + " return self.value + bb + arg\n" + //use scope variables + " def update(self, arg):\n" + + " global bb\n" + + " bb = self.value + arg\n" //update scope variable + ); + dynamic obj1 = _ps.Class1(20); + var result = obj1.call(10).As(); + Assert.AreEqual(130, result); + + obj1.update(10); + result = ps.Get("bb"); + Assert.AreEqual(30, result); + } + } + + /// + /// Import a python module into the session. + /// Equivalent to the Python "import" statement. + /// + [Test] + public void TestImportModule() + { + using (Py.GIL()) + { + dynamic sys = ps.Import("sys"); + Assert.IsTrue(ps.Contains("sys")); + + ps.Exec("sys.attr1 = 2"); + var value1 = ps.Eval("sys.attr1"); + var value2 = sys.attr1.As(); + Assert.AreEqual(2, value1); + Assert.AreEqual(2, value2); + + //import as + ps.Import("sys", "sys1"); + Assert.IsTrue(ps.Contains("sys1")); + } + } + + /// + /// Create a scope and import variables from a scope, + /// exec Python statements in the scope then discard it. + /// + [Test] + public void TestImportScope() + { + using (Py.GIL()) + { + ps.Set("bb", 100); + ps.Set("cc", 10); + + using (var scope = Py.CreateScope()) + { + scope.Import(ps, "ps"); + scope.Exec("aa = ps.bb + ps.cc + 3"); + var result = scope.Get("aa"); + Assert.AreEqual(113, result); + } + + Assert.IsFalse(ps.Contains("aa")); + } + } + + /// + /// Create a scope and import variables from a scope, + /// exec Python statements in the scope then discard it. + /// + [Test] + public void TestImportAllFromScope() + { + using (Py.GIL()) + { + ps.Set("bb", 100); + ps.Set("cc", 10); + + using (var scope = ps.NewScope()) + { + scope.Exec("aa = bb + cc + 3"); + var result = scope.Get("aa"); + Assert.AreEqual(113, result); + } + + Assert.IsFalse(ps.Contains("aa")); + } + } + + /// + /// Create a scope and import variables from a scope, + /// call the function imported. + /// + [Test] + public void TestImportScopeFunction() + { + using (Py.GIL()) + { + ps.Set("bb", 100); + ps.Set("cc", 10); + ps.Exec( + "def func1():\n" + + " return cc + bb\n"); + + using (PyScope scope = ps.NewScope()) + { + //'func1' is imported from the origion scope + scope.Exec( + "def func2():\n" + + " return func1() - cc - bb\n"); + dynamic func2 = scope.Get("func2"); + + var result1 = func2().As(); + Assert.AreEqual(0, result1); + + scope.Set("cc", 20);//it has no effect on the globals of 'func1' + var result2 = func2().As(); + Assert.AreEqual(-10, result2); + scope.Set("cc", 10); //rollback + + ps.Set("cc", 20); + var result3 = func2().As(); + Assert.AreEqual(10, result3); + ps.Set("cc", 10); //rollback + } + } + } + + /// + /// Import a python module into the session with a new name. + /// Equivalent to the Python "import .. as .." statement. + /// + [Test] + public void TestImportScopeByName() + { + using (Py.GIL()) + { + ps.Set("bb", 100); + + using (var scope = Py.CreateScope()) + { + scope.ImportAll("test"); + //scope.ImportModule("test"); + + Assert.IsTrue(scope.Contains("bb")); + } + } + } + + /// + /// Use the locals() and globals() method just like in python module + /// + [Test] + public void TestVariables() + { + using (Py.GIL()) + { + (ps.Variables() as dynamic)["ee"] = new PyInt(200); + var a0 = ps.Get("ee"); + Assert.AreEqual(200, a0); + + ps.Exec("locals()['ee'] = 210"); + var a1 = ps.Get("ee"); + Assert.AreEqual(210, a1); + + ps.Exec("globals()['ee'] = 220"); + var a2 = ps.Get("ee"); + Assert.AreEqual(220, a2); + + using (var item = ps.Variables()) + { + item["ee"] = new PyInt(230); + } + var a3 = ps.Get("ee"); + Assert.AreEqual(230, a3); + } + } + + /// + /// Share a pyscope by multiple threads. + /// + [Test] + public void TestThread() + { + //After the proposal here https://github.com/pythonnet/pythonnet/pull/419 complished, + //the BeginAllowThreads statement blow and the last EndAllowThreads statement + //should be removed. + dynamic _ps = ps; + var ts = PythonEngine.BeginAllowThreads(); + try + { + using (Py.GIL()) + { + _ps.res = 0; + _ps.bb = 100; + _ps.th_cnt = 0; + //add function to the scope + //can be call many times, more efficient than ast + ps.Exec( + "import threading\n"+ + "lock = threading.Lock()\n"+ + "def update():\n" + + " global res, th_cnt\n" + + " with lock:\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" + ); + } + int th_cnt = 100; + for (int i = 0; i < th_cnt; i++) + { + System.Threading.Thread th = new System.Threading.Thread(() => + { + using (Py.GIL()) + { + //ps.GetVariable("update")(); //call the scope function dynamicly + _ps.update(); + } + }); + th.Start(); + } + //equivalent to Thread.Join, make the main thread join the GIL competition + int cnt = 0; + while (cnt != th_cnt) + { + using (Py.GIL()) + { + cnt = ps.Get("th_cnt"); + } + Thread.Yield(); + } + using (Py.GIL()) + { + var result = ps.Get("res"); + Assert.AreEqual(101 * th_cnt, result); + } + } + finally + { + PythonEngine.EndAllowThreads(ts); + } + } + } +} diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs new file mode 100644 index 000000000..dcd539504 --- /dev/null +++ b/src/embed_tests/TestPyWith.cs @@ -0,0 +1,88 @@ +using System; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestPyWith + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + /// + /// Test that exception is raised in context manager that ignores it. + /// + [Test] + public void TestWithPositive() + { + var locals = new PyDict(); + + PythonEngine.Exec(@" +class CmTest: + def __enter__(self): + return self + def __exit__(self, t, v, tb): + # Exception not handled, return will be False + pass + def fail(self): + return 5 / 0 + +a = CmTest() +", null, locals.Handle); + + var a = locals.GetItem("a"); + + try + { + Py.With(a, cmTest => + { + cmTest.fail(); + }); + } + catch (PythonException e) + { + TestContext.Out.WriteLine(e.Message); + Assert.IsTrue(e.Message.Contains("ZeroDivisionError")); + } + } + + + /// + /// Test that exception is not raised in context manager that handles it + /// + [Test] + public void TestWithNegative() + { + var locals = new PyDict(); + + PythonEngine.Exec(@" +class CmTest: + def __enter__(self): + print('Enter') + return self + def __exit__(self, t, v, tb): + # Signal exception is handled by returning true + return True + def fail(self): + return 5 / 0 + +a = CmTest() +", null, locals.Handle); + + var a = locals.GetItem("a"); + Py.With(a, cmTest => + { + cmTest.fail(); + }); + } + } +} diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 01c6ae7e3..626e3c77f 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -109,18 +109,41 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { + // We needs to ensure that engine was started and shutdown at least once before setting dummy home. + // Otherwise engine will not run with dummy path with random problem. + if (!PythonEngine.IsInitialized) + { + PythonEngine.Initialize(); + } + + PythonEngine.Shutdown(); + + var pythonHomeBackup = PythonEngine.PythonHome; + var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); - Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); + + // Restoring valid pythonhome. + PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetPythonHomeTwice() { + // We needs to ensure that engine was started and shutdown at least once before setting dummy home. + // Otherwise engine will not run with dummy path with random problem. + if (!PythonEngine.IsInitialized) + { + PythonEngine.Initialize(); + } + PythonEngine.Shutdown(); + + var pythonHomeBackup = PythonEngine.PythonHome; + var pythonHome = "/dummypath/"; PythonEngine.PythonHome = "/dummypath2/"; @@ -129,11 +152,20 @@ public void SetPythonHomeTwice() Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); + + PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetProgramName() { + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + + var programNameBackup = PythonEngine.ProgramName; + var programName = "FooBar"; PythonEngine.ProgramName = programName; @@ -141,46 +173,45 @@ public void SetProgramName() Assert.AreEqual(programName, PythonEngine.ProgramName); PythonEngine.Shutdown(); + + PythonEngine.ProgramName = programNameBackup; } [Test] public void SetPythonPath() { - if (Runtime.Runtime.pyversion == "2.7") + PythonEngine.Initialize(); + + const string moduleName = "pytest"; + bool importShouldSucceed; + try { - // Assert.Skip outputs as a warning (ie. pending to fix) - Assert.Pass(); + Py.Import(moduleName); + importShouldSucceed = true; + } + catch + { + importShouldSucceed = false; } - PythonEngine.Initialize(); - string path = PythonEngine.PythonPath; - PythonEngine.Shutdown(); + string[] paths = Py.Import("sys").GetAttr("path").As(); + string path = string.Join(System.IO.Path.PathSeparator.ToString(), paths); - PythonEngine.ProgramName = path; - PythonEngine.Initialize(); + // path should not be set to PythonEngine.PythonPath here. + // PythonEngine.PythonPath gets the default module search path, not the full search path. + // The list sys.path is initialized with this value on interpreter startup; + // it can be (and usually is) modified later to change the search path for loading modules. + // See https://docs.python.org/3/c-api/init.html#c.Py_GetPath + // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. - Assert.AreEqual(path, PythonEngine.PythonPath); PythonEngine.Shutdown(); - } - [Test] - public void SetPythonPathExceptionOn27() - { - if (Runtime.Runtime.pyversion != "2.7") - { - Assert.Pass(); - } - - // Get previous path to avoid crashing Python + PythonEngine.PythonPath = path; PythonEngine.Initialize(); - string path = PythonEngine.PythonPath; - PythonEngine.Shutdown(); - - var ex = Assert.Throws(() => PythonEngine.PythonPath = "foo"); - Assert.AreEqual("Set PythonPath not supported on Python 2", ex.Message); - PythonEngine.Initialize(); Assert.AreEqual(path, PythonEngine.PythonPath); + if (importShouldSucceed) Py.Import(moduleName); + PythonEngine.Shutdown(); } } diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 5470b246f..3ab0f8106 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -40,5 +40,100 @@ public void TestNoError() var e = new PythonException(); // There is no PyErr to fetch Assert.AreEqual("", e.Message); } + + [Test] + public void TestPythonErrorTypeName() + { + try + { + var module = PythonEngine.ImportModule("really____unknown___module"); + Assert.Fail("Unknown module should not be loaded"); + } + catch (PythonException ex) + { + Assert.That(ex.PythonTypeName, Is.EqualTo("ModuleNotFoundError").Or.EqualTo("ImportError")); + } + } + + [Test] + public void TestPythonExceptionFormat() + { + try + { + PythonEngine.Exec("raise ValueError('Error!')"); + Assert.Fail("Exception should have been raised"); + } + catch (PythonException ex) + { + Assert.That(ex.Format(), Does.Contain("Traceback").And.Contains("(most recent call last):").And.Contains("ValueError: Error!")); + } + } + + [Test] + public void TestPythonExceptionFormatNoError() + { + var ex = new PythonException(); + Assert.AreEqual(ex.StackTrace, ex.Format()); + } + + [Test] + public void TestPythonExceptionFormatNoTraceback() + { + try + { + var module = PythonEngine.ImportModule("really____unknown___module"); + Assert.Fail("Unknown module should not be loaded"); + } + catch (PythonException ex) + { + // ImportError/ModuleNotFoundError do not have a traceback when not running in a script + Assert.AreEqual(ex.StackTrace, ex.Format()); + } + } + + [Test] + public void TestPythonExceptionFormatNormalized() + { + try + { + PythonEngine.Exec("a=b\n"); + } + catch (PythonException ex) + { + Assert.AreEqual("Traceback (most recent call last):\n File \"\", line 1, in \nNameError: name 'b' is not defined\n", ex.Format()); + } + } + + [Test] + public void TestPythonException_PyErr_NormalizeException() + { + using (var scope = Py.CreateScope()) + { + scope.Exec(@" +class TestException(NameError): + def __init__(self, val): + super().__init__(val) + x = int(val)"); + Assert.IsTrue(scope.TryGet("TestException", out PyObject type)); + + PyObject str = "dummy string".ToPython(); + IntPtr typePtr = type.Handle; + IntPtr strPtr = str.Handle; + IntPtr tbPtr = Runtime.Runtime.None.Handle; + Runtime.Runtime.XIncref(typePtr); + Runtime.Runtime.XIncref(strPtr); + Runtime.Runtime.XIncref(tbPtr); + Runtime.Runtime.PyErr_NormalizeException(ref typePtr, ref strPtr, ref tbPtr); + + using (PyObject typeObj = new PyObject(typePtr), strObj = new PyObject(strPtr), tbObj = new PyObject(tbPtr)) + { + // the type returned from PyErr_NormalizeException should not be the same type since a new + // exception was raised by initializing the exception + Assert.AreNotEqual(type.Handle, typePtr); + // the message should now be the string from the throw exception during normalization + Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); + } + } + } } } diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 22e6db0a9..cde5dd6fa 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,15 +1,31 @@ using System; +using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; namespace Python.EmbeddingTest { public class TestRuntime { + [OneTimeSetUp] + public void SetUp() + { + // We needs to ensure that no any engines are running. + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + } + [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); // In case another test left it on. + if (Runtime.Runtime.Py_IsInitialized() == 1) + { + Runtime.Runtime.PyGILState_Ensure(); + } + Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); @@ -47,5 +63,52 @@ public static void RefCountTest() Runtime.Runtime.Py_Finalize(); } + + [Test] + public static void PyCheck_Iter_PyObject_IsIterable_Test() + { + Runtime.Runtime.Py_Initialize(); + + // Tests that a python list is an iterable, but not an iterator + var pyList = Runtime.Runtime.PyList_New(0); + Assert.IsFalse(Runtime.Runtime.PyIter_Check(pyList)); + Assert.IsTrue(Runtime.Runtime.PyObject_IsIterable(pyList)); + + // Tests that a python list iterator is both an iterable and an iterator + var pyListIter = Runtime.Runtime.PyObject_GetIter(pyList); + Assert.IsTrue(Runtime.Runtime.PyObject_IsIterable(pyListIter)); + Assert.IsTrue(Runtime.Runtime.PyIter_Check(pyListIter)); + + // Tests that a python float is neither an iterable nor an iterator + var pyFloat = Runtime.Runtime.PyFloat_FromDouble(2.73); + Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(pyFloat)); + Assert.IsFalse(Runtime.Runtime.PyIter_Check(pyFloat)); + + Runtime.Runtime.Py_Finalize(); + } + + [Test] + public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() + { + Runtime.Runtime.Py_Initialize(); + + // Create an instance of threading.Lock, which is one of the very few types that does not have the + // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. + var threading = Runtime.Runtime.PyImport_ImportModule("threading"); + Exceptions.ErrorCheck(threading); + var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); + Exceptions.ErrorCheck(threadingDict); + var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); + if (lockType == IntPtr.Zero) + throw new KeyNotFoundException("class 'Lock' was not found in 'threading'"); + + var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0)); + Exceptions.ErrorCheck(lockInstance); + + Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); + Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); + + Runtime.Runtime.Py_Finalize(); + } } } diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs new file mode 100644 index 000000000..43155e1bf --- /dev/null +++ b/src/embed_tests/TestTypeManager.cs @@ -0,0 +1,65 @@ +using NUnit.Framework; +using Python.Runtime; +using Python.Runtime.Platform; +using System.Runtime.InteropServices; + +namespace Python.EmbeddingTest +{ + class TestTypeManager + { + [SetUp] + public static void Init() + { + Runtime.Runtime.Initialize(); + } + + [TearDown] + public static void Fini() + { + Runtime.Runtime.Shutdown(); + } + + [Test] + public static void TestNativeCode() + { + Assert.That(() => { var _ = NativeCodePageHelper.NativeCode.Active; }, Throws.Nothing); + Assert.That(NativeCodePageHelper.NativeCode.Active.Code.Length, Is.GreaterThan(0)); + } + + [Test] + public static void TestMemoryMapping() + { + Assert.That(() => { var _ = NativeCodePageHelper.CreateMemoryMapper(); }, Throws.Nothing); + var mapper = NativeCodePageHelper.CreateMemoryMapper(); + + // Allocate a read-write page. + int len = 12; + var page = mapper.MapWriteable(len); + Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); + Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); + + // Mark it read-execute. We can still read, haven't changed any values. + mapper.SetReadExec(page, len); + Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); + + // Test that we can't write to the protected page. + // + // We can't actually test access protection under Microsoft + // versions of .NET, because AccessViolationException is assumed to + // mean we're in a corrupted state: + // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception + // + // We can test under Mono but it throws NRE instead of AccessViolationException. + // + // We can't use compiler flags because we compile with MONO_LINUX + // while running on the Microsoft .NET Core during continuous + // integration tests. + /* if (System.Type.GetType ("Mono.Runtime") != null) + { + // Mono throws NRE instead of AccessViolationException for some reason. + Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); + Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); + } */ + } + } +} diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 94397072a..81345cee7 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -12,13 +12,23 @@ public class DynamicTest [SetUp] public void SetUp() { - _gs = Py.GIL(); + try { + _gs = Py.GIL(); + } catch (Exception e) { + Console.WriteLine($"exception in SetUp: {e}"); + throw; + } } [TearDown] public void Dispose() { - _gs.Dispose(); + try { + _gs.Dispose(); + } catch(Exception e) { + Console.WriteLine($"exception in TearDown: {e}"); + throw; + } } /// @@ -103,7 +113,7 @@ public void PassObjectInPython() Assert.AreEqual(sys.testattr3.ToString(), "True"); // Compare in .NET - Assert.AreEqual(sys.testattr1, sys.testattr2); + Assert.IsTrue(sys.testattr1.Equals(sys.testattr2)); } /// @@ -118,14 +128,15 @@ public void PassPyObjectInNet() sys.testattr2 = sys.testattr1; // Compare in Python - PyObject res = PythonEngine.RunString( + PythonEngine.RunSimpleString( "import sys\n" + "sys.testattr3 = sys.testattr1 is sys.testattr2\n" ); + Assert.AreEqual(sys.testattr3.ToString(), "True"); // Compare in .NET - Assert.AreEqual(sys.testattr1, sys.testattr2); + Assert.IsTrue(sys.testattr1.Equals(sys.testattr2)); } } } diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 4cb01d3be..590eaef8c 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,6 @@ - + - - + + + diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index 3bb9a34d6..24f31acff 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -30,12 +30,13 @@ public void SetUp() /* Append the tests directory to sys.path * using reflection to circumvent the private * modifiers placed on most Runtime methods. */ - const string s = "../fixtures"; - string testPath = Path.Combine(TestContext.CurrentContext.TestDirectory, s); + string testPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "fixtures"); + TestContext.Out.WriteLine(testPath); IntPtr str = Runtime.Runtime.PyString_FromString(testPath); - IntPtr path = Runtime.Runtime.PySys_GetObject("path"); + BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.XDecref(str); } [TearDown] @@ -79,5 +80,28 @@ public void TestCastGlobalVar() Assert.AreEqual("2", foo.FOO.ToString()); Assert.AreEqual("2", foo.test_foo().ToString()); } + + [Test] + public void BadAssembly() + { + string path; + if (Python.Runtime.Runtime.IsWindows) + { + path = @"C:\Windows\System32\kernel32.dll"; + } + else + { + Assert.Pass("TODO: add bad assembly location for other platforms"); + return; + } + + string code = $@" +import clr +clr.AddReference('{path}') +"; + + var error = Assert.Throws(() => PythonEngine.Exec(code)); + Assert.AreEqual(nameof(FileLoadException), error.PythonTypeName); + } } } diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 2f9aae2c7..c774680dd 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -24,9 +24,11 @@ public static void StartAndStopTwice() public static void LoadDefaultArgs() { using (new PythonEngine()) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Assert.AreNotEqual(0, argv.Length()); + using(var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) + { + Assert.AreNotEqual(0, argv.Length()); + } } } @@ -35,10 +37,12 @@ public static void LoadSpecificArgs() { var args = new[] { "test1", "test2" }; using (new PythonEngine(args)) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Assert.AreEqual(args[0], argv[0].ToString()); - Assert.AreEqual(args[1], argv[1].ToString()); + using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) + { + Assert.AreEqual(args[0], argv[0].ToString()); + Assert.AreEqual(args[1], argv[1].ToString()); + } } } @@ -74,5 +78,106 @@ public void ReInitialize() } PythonEngine.Shutdown(); } + + [Test] + public void TestScopeIsShutdown() + { + PythonEngine.Initialize(); + var scope = PyScopeManager.Global.Create("test"); + PythonEngine.Shutdown(); + Assert.That(PyScopeManager.Global.Contains("test"), Is.False); + } + + /// + /// Helper for testing the shutdown handlers. + /// + int shutdown_count = 0; + void OnShutdownIncrement() + { + shutdown_count++; + } + void OnShutdownDouble() + { + shutdown_count *= 2; + } + + /// + /// Test the shutdown handlers. + /// + [Test] + public void ShutdownHandlers() + { + // Test we can run one shutdown handler. + shutdown_count = 0; + PythonEngine.Initialize(); + PythonEngine.AddShutdownHandler(OnShutdownIncrement); + PythonEngine.Shutdown(); + Assert.That(shutdown_count, Is.EqualTo(1)); + + // Test we can run multiple shutdown handlers in the right order. + shutdown_count = 4; + PythonEngine.Initialize(); + PythonEngine.AddShutdownHandler(OnShutdownIncrement); + PythonEngine.AddShutdownHandler(OnShutdownDouble); + PythonEngine.Shutdown(); + // Correct: 4 * 2 + 1 = 9 + // Wrong: (4 + 1) * 2 = 10 + Assert.That(shutdown_count, Is.EqualTo(9)); + + // Test we can remove shutdown handlers, handling duplicates. + shutdown_count = 4; + PythonEngine.Initialize(); + PythonEngine.AddShutdownHandler(OnShutdownIncrement); + PythonEngine.AddShutdownHandler(OnShutdownIncrement); + PythonEngine.AddShutdownHandler(OnShutdownDouble); + PythonEngine.AddShutdownHandler(OnShutdownIncrement); + PythonEngine.AddShutdownHandler(OnShutdownDouble); + PythonEngine.RemoveShutdownHandler(OnShutdownDouble); + PythonEngine.Shutdown(); + // Correct: (4 + 1) * 2 + 1 + 1 = 12 + // Wrong: (4 * 2) + 1 + 1 + 1 = 11 + Assert.That(shutdown_count, Is.EqualTo(12)); + } + + [Test] + public static void TestRunExitFuncs() + { + if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal) + { + // If the runtime using the normal mode, + // callback registered by atexit will be called after we release the clr information, + // thus there's no chance we can check it here. + Assert.Ignore("Skip on normal mode"); + } + Runtime.Runtime.Initialize(); + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + string msg = e.ToString(); + Runtime.Runtime.Shutdown(); + + if (e.IsMatches(Exceptions.ImportError)) + { + Assert.Ignore("no atexit module"); + } + else + { + Assert.Fail(msg); + } + return; + } + bool called = false; + Action callback = () => + { + called = true; + }; + atexit.InvokeMethod("register", callback.ToPython()); + Runtime.Runtime.Shutdown(); + Assert.True(called); + } } } diff --git a/src/embed_tests/pyrunstring.cs b/src/embed_tests/pyrunstring.cs index 81a1b07ca..07875a2a8 100644 --- a/src/embed_tests/pyrunstring.cs +++ b/src/embed_tests/pyrunstring.cs @@ -57,5 +57,20 @@ public void TestExec() object c = locals.GetItem("c").AsManagedObject(typeof(int)); Assert.AreEqual(111, c); } + + [Test] + public void TestExec2() + { + string code = @" +class Test1(): + pass + +class Test2(): + def __init__(self): + Test1() + +Test2()"; + PythonEngine.Exec(code); + } } } diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c index 4e8027e3a..cdfd89342 100644 --- a/src/monoclr/clrmod.c +++ b/src/monoclr/clrmod.c @@ -1,70 +1,215 @@ -#include "pynetclr.h" +// #define Py_LIMITED_API 0x03050000 +#include -/* List of functions defined in the module */ -static PyMethodDef clr_methods[] = { - {NULL, NULL, 0, NULL} /* Sentinel */ -}; +#include "stdlib.h" -PyDoc_STRVAR(clr_module_doc, - "clr facade module to initialize the CLR. It's later " - "replaced by the real clr module. This module has a facade " - "attribute to make it distinguishable from the real clr module." -); +#define MONO_VERSION "v4.0.30319.1" +#define MONO_DOMAIN "Python" -static PyNet_Args *pn_args; -char **environ = NULL; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef clrdef = { - PyModuleDef_HEAD_INIT, - "clr", /* m_name */ - clr_module_doc, /* m_doc */ - -1, /* m_size */ - clr_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL, /* m_free */ -}; +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include "dirent.h" +#include "dlfcn.h" +#include "libgen.h" +#include "alloca.h" #endif -static PyObject *_initclr() +typedef struct { - PyObject *m; + MonoDomain *domain; + MonoAssembly *pr_assm; + MonoMethod *shutdown; + const char *pr_file; + char *error; + char *init_name; + char *shutdown_name; + PyObject *module; +} PyNet_Args; - /* Create the module and add the functions */ -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&clrdef); -#else - m = Py_InitModule3("clr", clr_methods, clr_module_doc); -#endif - if (m == NULL) - return NULL; - PyModule_AddObject(m, "facade", Py_True); - Py_INCREF(Py_True); +PyNet_Args *PyNet_Init(void); +static PyNet_Args *pn_args; - pn_args = PyNet_Init(1); +PyMODINIT_FUNC +PyInit_clr(void) +{ + pn_args = PyNet_Init(); if (pn_args->error) { return NULL; } - if (NULL != pn_args->module) - return pn_args->module; + return pn_args->module; +} + +void PyNet_Finalize(PyNet_Args *); +void main_thread_handler(PyNet_Args *user_data); + +// initialize Mono and PythonNet +PyNet_Args *PyNet_Init() +{ + PyObject *pn_module; + PyObject *pn_path; + PyNet_Args *pn_args; + pn_args = (PyNet_Args *)malloc(sizeof(PyNet_Args)); + + pn_module = PyImport_ImportModule("pythonnet"); + if (pn_module == NULL) + { + pn_args->error = "Failed to import pythonnet"; + return pn_args; + } + + pn_path = PyObject_CallMethod(pn_module, "get_assembly_path", NULL); + if (pn_path == NULL) + { + Py_DecRef(pn_module); + pn_args->error = "Failed to get assembly path"; + return pn_args; + } + + pn_args->pr_file = PyUnicode_AsUTF8(pn_path); + pn_args->error = NULL; + pn_args->shutdown = NULL; + pn_args->module = NULL; + +#ifdef __linux__ + // Force preload libmono-2.0 as global. Without this, on some systems + // symbols from libmono are not found by libmononative (which implements + // some of the System.* namespaces). Since the only happened on Linux so + // far, we hardcode the library name, load the symbols into the global + // namespace and leak the handle. + dlopen("libmono-2.0.so", RTLD_LAZY | RTLD_GLOBAL); +#endif + + pn_args->init_name = "Python.Runtime:InitExt()"; + pn_args->shutdown_name = "Python.Runtime:Shutdown()"; + + pn_args->domain = mono_jit_init_version(MONO_DOMAIN, MONO_VERSION); + + // XXX: Skip setting config for now, should be derived from pr_file + // mono_domain_set_config(pn_args->domain, ".", "Python.Runtime.dll.config"); + + /* + * Load the default Mono configuration file, this is needed + * if you are planning on using the dllmaps defined on the + * system configuration + */ + mono_config_parse(NULL); + + main_thread_handler(pn_args); - return m; + if (pn_args->error != NULL) + { + PyErr_SetString(PyExc_ImportError, pn_args->error); + } + return pn_args; } -#if PY_MAJOR_VERSION >= 3 -PyMODINIT_FUNC -PyInit_clr(void) +char *PyNet_ExceptionToString(MonoObject *e); + +// Shuts down PythonNet and cleans up Mono +void PyNet_Finalize(PyNet_Args *pn_args) +{ + MonoObject *exception = NULL; + + if (pn_args->shutdown) + { + mono_runtime_invoke(pn_args->shutdown, NULL, NULL, &exception); + if (exception) + { + pn_args->error = PyNet_ExceptionToString(exception); + } + pn_args->shutdown = NULL; + } + + if (pn_args->domain) + { + mono_jit_cleanup(pn_args->domain); + pn_args->domain = NULL; + } + free(pn_args); +} + +MonoMethod *getMethodFromClass(MonoClass *cls, char *name) { - return _initclr(); + MonoMethodDesc *mdesc; + MonoMethod *method; + + mdesc = mono_method_desc_new(name, 1); + method = mono_method_desc_search_in_class(mdesc, cls); + mono_method_desc_free(mdesc); + + return method; } -#else -PyMODINIT_FUNC -initclr(void) + +void main_thread_handler(PyNet_Args *user_data) +{ + PyNet_Args *pn_args = user_data; + MonoMethod *init; + MonoImage *pr_image; + MonoClass *pythonengine; + MonoObject *exception = NULL; + MonoObject *init_result; + + pn_args->pr_assm = mono_domain_assembly_open(pn_args->domain, pn_args->pr_file); + if (!pn_args->pr_assm) + { + pn_args->error = "Unable to load assembly"; + return; + } + + pr_image = mono_assembly_get_image(pn_args->pr_assm); + if (!pr_image) + { + pn_args->error = "Unable to get image"; + return; + } + + pythonengine = mono_class_from_name(pr_image, "Python.Runtime", "PythonEngine"); + if (!pythonengine) + { + pn_args->error = "Unable to load class PythonEngine from Python.Runtime"; + return; + } + + init = getMethodFromClass(pythonengine, pn_args->init_name); + if (!init) + { + pn_args->error = "Unable to fetch Init method from PythonEngine"; + return; + } + + pn_args->shutdown = getMethodFromClass(pythonengine, pn_args->shutdown_name); + if (!pn_args->shutdown) + { + pn_args->error = "Unable to fetch shutdown method from PythonEngine"; + return; + } + + init_result = mono_runtime_invoke(init, NULL, NULL, &exception); + if (exception) + { + pn_args->error = PyNet_ExceptionToString(exception); + return; + } + + pn_args->module = *(PyObject**)mono_object_unbox(init_result); +} + +// Get string from a Mono exception +char *PyNet_ExceptionToString(MonoObject *e) { - _initclr(); + MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", 0 /*FALSE*/); + MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); + mono_method_desc_free(mdesc); + + mmethod = mono_object_get_virtual_method(e, mmethod); + MonoString *monoString = (MonoString*) mono_runtime_invoke(mmethod, e, NULL, NULL); + mono_runtime_invoke(mmethod, e, NULL, NULL); + return mono_string_to_utf8(monoString); } -#endif diff --git a/src/monoclr/pynetclr.h b/src/monoclr/pynetclr.h deleted file mode 100644 index c5e181156..000000000 --- a/src/monoclr/pynetclr.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef PYNET_CLR_H -#define PYNET_CLR_H - -#include -#include -#include -#include -#include -#include -#include - -#define MONO_VERSION "v4.0.30319.1" -#define MONO_DOMAIN "Python.Runtime" -#define PR_ASSEMBLY "Python.Runtime.dll" - -typedef struct -{ - MonoDomain *domain; - MonoAssembly *pr_assm; - MonoMethod *shutdown; - char *pr_file; - char *error; - char *init_name; - char *shutdown_name; - PyObject *module; -} PyNet_Args; - -PyNet_Args *PyNet_Init(int); -void PyNet_Finalize(PyNet_Args *); -void main_thread_handler(gpointer user_data); -char *PyNet_ExceptionToString(MonoObject *); - -#endif // PYNET_CLR_H diff --git a/src/monoclr/pynetinit.c b/src/monoclr/pynetinit.c deleted file mode 100644 index 8b49ddae3..000000000 --- a/src/monoclr/pynetinit.c +++ /dev/null @@ -1,252 +0,0 @@ -#include "pynetclr.h" -#include "stdlib.h" - -#ifndef _WIN32 -#include "dirent.h" -#include "dlfcn.h" -#include "libgen.h" -#include "alloca.h" -#endif - - -// initialize Mono and PythonNet -PyNet_Args *PyNet_Init(int ext) -{ - PyNet_Args *pn_args; - pn_args = (PyNet_Args *)malloc(sizeof(PyNet_Args)); - pn_args->pr_file = PR_ASSEMBLY; - pn_args->error = NULL; - pn_args->shutdown = NULL; - pn_args->module = NULL; - - if (ext == 0) - { - pn_args->init_name = "Python.Runtime:Initialize()"; - } - else - { - pn_args->init_name = "Python.Runtime:InitExt()"; - } - pn_args->shutdown_name = "Python.Runtime:Shutdown()"; - - pn_args->domain = mono_jit_init_version(MONO_DOMAIN, MONO_VERSION); - mono_domain_set_config(pn_args->domain, ".", "Python.Runtime.dll.config"); - - /* - * Load the default Mono configuration file, this is needed - * if you are planning on using the dllmaps defined on the - * system configuration - */ - mono_config_parse(NULL); - - /* I can't use this call to run the main_thread_handler. The function - * runs it in another thread but *this* thread holds the Python - * import lock -> DEAD LOCK. - * - * mono_runtime_exec_managed_code(pn_args->domain, main_thread_handler, - * pn_args); - */ - - main_thread_handler(pn_args); - - if (pn_args->error != NULL) - { - PyErr_SetString(PyExc_ImportError, pn_args->error); - } - return pn_args; -} - -// Shuts down PythonNet and cleans up Mono -void PyNet_Finalize(PyNet_Args *pn_args) -{ - MonoObject *exception = NULL; - - if (pn_args->shutdown) - { - mono_runtime_invoke(pn_args->shutdown, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - } - pn_args->shutdown = NULL; - } - - if (pn_args->domain) - { - mono_jit_cleanup(pn_args->domain); - pn_args->domain = NULL; - } - free(pn_args); -} - -MonoMethod *getMethodFromClass(MonoClass *cls, char *name) -{ - MonoMethodDesc *mdesc; - MonoMethod *method; - - mdesc = mono_method_desc_new(name, 1); - method = mono_method_desc_search_in_class(mdesc, cls); - mono_method_desc_free(mdesc); - - return method; -} - -void main_thread_handler(gpointer user_data) -{ - PyNet_Args *pn_args = (PyNet_Args *)user_data; - MonoMethod *init; - MonoImage *pr_image; - MonoClass *pythonengine; - MonoObject *exception = NULL; - MonoObject *init_result; - -#ifndef _WIN32 - // Get the filename of the python shared object and set - // LD_LIBRARY_PATH so Mono can find it. - Dl_info dlinfo = {0}; - if (0 != dladdr(&Py_Initialize, &dlinfo)) - { - char *fname = alloca(strlen(dlinfo.dli_fname) + 1); - strcpy(fname, dlinfo.dli_fname); - char *py_libdir = dirname(fname); - char *ld_library_path = getenv("LD_LIBRARY_PATH"); - if (NULL == ld_library_path) - { - setenv("LD_LIBRARY_PATH", py_libdir, 1); - } - else - { - char *new_ld_library_path = alloca(strlen(py_libdir) - + strlen(ld_library_path) - + 2); - strcpy(new_ld_library_path, py_libdir); - strcat(new_ld_library_path, ":"); - strcat(new_ld_library_path, ld_library_path); - setenv("LD_LIBRARY_PATH", py_libdir, 1); - } - } - - //get python path system variable - PyObject *syspath = PySys_GetObject("path"); - char *runtime_full_path = (char*) malloc(1024); - const char *slash = "/"; - int found = 0; - - int ii = 0; - for (ii = 0; ii < PyList_Size(syspath); ++ii) - { -#if PY_MAJOR_VERSION >= 3 - Py_ssize_t wlen; - wchar_t *wstr = PyUnicode_AsWideCharString(PyList_GetItem(syspath, ii), &wlen); - char *pydir = (char*)malloc(wlen + 1); - size_t mblen = wcstombs(pydir, wstr, wlen + 1); - if (mblen > wlen) - pydir[wlen] = '\0'; - PyMem_Free(wstr); -#else - const char *pydir = PyString_AsString(PyList_GetItem(syspath, ii)); -#endif - char *curdir = (char*) malloc(1024); - strncpy(curdir, strlen(pydir) > 0 ? pydir : ".", 1024); - strncat(curdir, slash, 1024); - -#if PY_MAJOR_VERSION >= 3 - free(pydir); -#endif - - //look in this directory for the pn_args->pr_file - DIR *dirp = opendir(curdir); - if (dirp != NULL) - { - struct dirent *dp; - while ((dp = readdir(dirp)) != NULL) - { - if (strcmp(dp->d_name, pn_args->pr_file) == 0) - { - strcpy(runtime_full_path, curdir); - strcat(runtime_full_path, pn_args->pr_file); - found = 1; - break; - } - } - closedir(dirp); - } - free(curdir); - - if (found) - { - pn_args->pr_file = runtime_full_path; - break; - } - } - - if (!found) - { - fprintf(stderr, "Could not find assembly %s. \n", pn_args->pr_file); - return; - } -#endif - - pn_args->pr_assm = mono_domain_assembly_open(pn_args->domain, pn_args->pr_file); - if (!pn_args->pr_assm) - { - pn_args->error = "Unable to load assembly"; - return; - } -#ifndef _WIN32 - free(runtime_full_path); -#endif - - pr_image = mono_assembly_get_image(pn_args->pr_assm); - if (!pr_image) - { - pn_args->error = "Unable to get image"; - return; - } - - pythonengine = mono_class_from_name(pr_image, "Python.Runtime", "PythonEngine"); - if (!pythonengine) - { - pn_args->error = "Unable to load class PythonEngine from Python.Runtime"; - return; - } - - init = getMethodFromClass(pythonengine, pn_args->init_name); - if (!init) - { - pn_args->error = "Unable to fetch Init method from PythonEngine"; - return; - } - - pn_args->shutdown = getMethodFromClass(pythonengine, pn_args->shutdown_name); - if (!pn_args->shutdown) - { - pn_args->error = "Unable to fetch shutdown method from PythonEngine"; - return; - } - - init_result = mono_runtime_invoke(init, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - return; - } - -#if PY_MAJOR_VERSION >= 3 - if (NULL != init_result) - pn_args->module = *(PyObject**)mono_object_unbox(init_result); -#endif -} - -// Get string from a Mono exception -char *PyNet_ExceptionToString(MonoObject *e) -{ - MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", FALSE); - MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); - mono_method_desc_free(mdesc); - - mmethod = mono_object_get_virtual_method(e, mmethod); - MonoString *monoString = (MonoString*) mono_runtime_invoke(mmethod, e, NULL, NULL); - mono_runtime_invoke(mmethod, e, NULL, NULL); - return mono_string_to_utf8(monoString); -} diff --git a/src/perf_tests/BaselineComparisonBenchmarkBase.cs b/src/perf_tests/BaselineComparisonBenchmarkBase.cs new file mode 100644 index 000000000..2388e3982 --- /dev/null +++ b/src/perf_tests/BaselineComparisonBenchmarkBase.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +using Python.Runtime; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonBenchmarkBase + { + public BaselineComparisonBenchmarkBase() + { + Console.WriteLine($"CWD: {Environment.CurrentDirectory}"); + Console.WriteLine($"Using Python.Runtime from {typeof(PythonEngine).Assembly.Location} {typeof(PythonEngine).Assembly.GetName()}"); + + try { + PythonEngine.Initialize(); + Console.WriteLine("Python Initialized"); + if (PythonEngine.BeginAllowThreads() == IntPtr.Zero) + throw new PythonException(); + Console.WriteLine("Threading enabled"); + } + catch (Exception e) { + Console.WriteLine(e); + throw; + } + } + + static BaselineComparisonBenchmarkBase() + { + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + { + throw new ArgumentException( + "Required environment variable is missing", + BaselineComparisonConfig.EnvironmentVariableName); + } + + Console.WriteLine("Preloading " + pythonRuntimeDll); + Assembly.LoadFrom(pythonRuntimeDll); + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) { + if (assembly.FullName.StartsWith("Python.Runtime")) + Console.WriteLine(assembly.Location); + foreach(var dependency in assembly.GetReferencedAssemblies()) + if (dependency.FullName.Contains("Python.Runtime")) { + Console.WriteLine($"{assembly} -> {dependency}"); + } + } + + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + if (!args.Name.StartsWith("Python.Runtime")) + return null; + + var preloaded = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "Python.Runtime"); + if (preloaded != null) return preloaded; + + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + return null; + + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs new file mode 100644 index 000000000..649bb56fd --- /dev/null +++ b/src/perf_tests/BaselineComparisonConfig.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Horology; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonConfig : ManualConfig + { + public const string EnvironmentVariableName = "PythonRuntimeDLL"; + + public BaselineComparisonConfig() + { + this.Options |= ConfigOptions.DisableOptimizationsValidator; + + string deploymentRoot = BenchmarkTests.DeploymentRoot; + + var baseJob = Job.Default + .WithLaunchCount(1) + .WithWarmupCount(3) + .WithMaxIterationCount(100) + .WithIterationTime(TimeInterval.FromMilliseconds(100)); + this.Add(baseJob + .WithId("baseline") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "baseline", "Python.Runtime.dll")) + .WithBaseline(true)); + this.Add(baseJob + .WithId("new") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "new", "Python.Runtime.dll"))); + } + + static BaselineComparisonConfig() { + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + Console.WriteLine(args.Name); + if (!args.Name.StartsWith("Python.Runtime")) + return null; + string pythonRuntimeDll = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + pythonRuntimeDll = Path.Combine(BenchmarkTests.DeploymentRoot, "baseline", "Python.Runtime.dll"); + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs new file mode 100644 index 000000000..6e0afca69 --- /dev/null +++ b/src/perf_tests/BenchmarkTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Reflection; + +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using NUnit.Framework; + +namespace Python.PerformanceTests +{ + public class BenchmarkTests + { + Summary summary; + + [OneTimeSetUp] + public void SetUp() + { + Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new"); + this.summary = BenchmarkRunner.Run(); + Assert.IsNotEmpty(this.summary.Reports); + Assert.IsTrue( + condition: this.summary.Reports.All(r => r.Success), + message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above."); + } + + [Test] + public void ReadInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57); + } + + [Test] + public void WriteInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57); + } + + static double GetOptimisticPerfRatio( + IReadOnlyList reports, + [CallerMemberName] string methodName = null) + { + reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray(); + if (reports.Count == 0) + throw new ArgumentException( + message: $"No reports found for {methodName}. " + + "You have to match test method name to benchmark method name or " + + "pass benchmark method name explicitly", + paramName: nameof(methodName)); + + var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics; + var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics; + + double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5; + + return newTimeOptimistic / baseline.Mean; + } + + public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + + public static void AssertPerformanceIsBetterOrSame( + double actual, double target, + double wiggleRoom = 1.1, [CallerMemberName] string testName = null) { + double threshold = target * wiggleRoom; + Assert.LessOrEqual(actual, threshold, + $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})" + + ": perf result is higher than the failure threshold."); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj new file mode 100644 index 000000000..22783e595 --- /dev/null +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -0,0 +1,40 @@ + + + + net472 + false + x64;x86 + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + compile + + + + + + + + + + + + + + + + + + diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs new file mode 100644 index 000000000..ef668a911 --- /dev/null +++ b/src/perf_tests/PythonCallingNetBenchmark.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using BenchmarkDotNet.Attributes; +using Python.Runtime; + +namespace Python.PerformanceTests +{ + [Config(typeof(BaselineComparisonConfig))] + public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase + { + [Benchmark] + public void ReadInt64Property() + { + using (Py.GIL()) + { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(50000): + s += a.{nameof(NetObject.LongProperty)} +", locals: locals.Handle); + } + } + + [Benchmark] + public void WriteInt64Property() { + using (Py.GIL()) { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(50000): + a.{nameof(NetObject.LongProperty)} += i +", locals: locals.Handle); + } + } + } + + class NetObject + { + public long LongProperty { get; set; } = 42; + } +} diff --git a/src/python_tests_runner/Python.PythonTestsRunner.csproj b/src/python_tests_runner/Python.PythonTestsRunner.csproj new file mode 100644 index 000000000..2d6544614 --- /dev/null +++ b/src/python_tests_runner/Python.PythonTestsRunner.csproj @@ -0,0 +1,25 @@ + + + + net472;netcoreapp3.1 + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + 1.0.0 + all + runtime; build; native; contentfiles; analyzers + + + + diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs new file mode 100644 index 000000000..79b15700e --- /dev/null +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.PythonTestsRunner +{ + public class PythonTestRunner + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + /// + /// Selects the Python tests to be run as embedded tests. + /// + /// + static IEnumerable PythonTestCases() + { + // Add the test that you want to debug here. + yield return new[] { "test_enum", "test_enum_standard_attrs" }; + yield return new[] { "test_generic", "test_missing_generic_type" }; + } + + /// + /// Runs a test in src/tests/*.py as an embedded test. This facilitates debugging. + /// + /// The file name without extension + /// The name of the test method + [TestCaseSource(nameof(PythonTestCases))] + public void RunPythonTest(string testFile, string testName) + { + // Find the tests directory + string folder = typeof(PythonTestRunner).Assembly.Location; + while (Path.GetFileName(folder) != "src") + { + folder = Path.GetDirectoryName(folder); + } + folder = Path.Combine(folder, "tests"); + string path = Path.Combine(folder, testFile + ".py"); + if (!File.Exists(path)) throw new FileNotFoundException("Cannot find test file", path); + + // We could use 'import' below, but importlib gives more helpful error messages than 'import' + // https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly + // Because the Python tests sometimes have relative imports, the module name must be inside the tests package + PythonEngine.Exec($@" +import sys +import os +sys.path.append(os.path.dirname(r'{folder}')) +sys.path.append(os.path.join(r'{folder}', 'fixtures')) +import clr +clr.AddReference('Python.Test') +import tests +module_name = 'tests.{testFile}' +file_path = r'{path}' +import importlib.util +spec = importlib.util.spec_from_file_location(module_name, file_path) +module = importlib.util.module_from_spec(spec) +sys.modules[module_name] = module +try: + spec.loader.exec_module(module) +except ImportError as error: + raise ImportError(str(error) + ' when sys.path=' + os.pathsep.join(sys.path)) +module.{testName}() +"); + } + } +} diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs new file mode 100644 index 000000000..a9ea327e9 --- /dev/null +++ b/src/runtime/BorrowedReference.cs @@ -0,0 +1,40 @@ +namespace Python.Runtime +{ + using System; + /// + /// Represents a reference to a Python object, that is being lent, and + /// can only be safely used until execution returns to the caller. + /// + readonly ref struct BorrowedReference + { + readonly IntPtr pointer; + public bool IsNull => this.pointer == IntPtr.Zero; + + + /// Gets a raw pointer to the Python object + public IntPtr DangerousGetAddress() + => this.IsNull ? throw new NullReferenceException() : this.pointer; + + /// + /// Creates new instance of from raw pointer. Unsafe. + /// + public BorrowedReference(IntPtr pointer) + { + this.pointer = pointer; + } + + public static bool operator ==(BorrowedReference a, BorrowedReference b) + => a.pointer == b.pointer; + public static bool operator !=(BorrowedReference a, BorrowedReference b) + => a.pointer != b.pointer; + + public override bool Equals(object obj) { + if (obj is IntPtr ptr) + return ptr == pointer; + + return false; + } + + public override int GetHashCode() => pointer.GetHashCode(); + } +} diff --git a/src/runtime/Codecs/DecoderGroup.cs b/src/runtime/Codecs/DecoderGroup.cs new file mode 100644 index 000000000..cc511ed50 --- /dev/null +++ b/src/runtime/Codecs/DecoderGroup.cs @@ -0,0 +1,75 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a group of s. Useful to group them by priority. + /// + public sealed class DecoderGroup: IPyObjectDecoder, IEnumerable + { + readonly List decoders = new List(); + + /// + /// Add specified decoder to the group + /// + public void Add(IPyObjectDecoder item) + { + if (item is null) throw new ArgumentNullException(nameof(item)); + + this.decoders.Add(item); + } + /// + /// Remove all decoders from the group + /// + public void Clear() => this.decoders.Clear(); + + /// + public bool CanDecode(PyObject objectType, Type targetType) + => this.decoders.Any(decoder => decoder.CanDecode(objectType, targetType)); + /// + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj is null) throw new ArgumentNullException(nameof(pyObj)); + + var decoder = this.GetDecoder(pyObj.GetPythonType(), typeof(T)); + if (decoder is null) + { + value = default; + return false; + } + return decoder.TryDecode(pyObj, out value); + } + + /// + public IEnumerator GetEnumerator() => this.decoders.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => this.decoders.GetEnumerator(); + } + + public static class DecoderGroupExtensions + { + /// + /// Gets a concrete instance of + /// (potentially selecting one from a collection), + /// that can decode from to , + /// or null if a matching decoder can not be found. + /// + public static IPyObjectDecoder GetDecoder( + this IPyObjectDecoder decoder, + PyObject objectType, Type targetType) + { + if (decoder is null) throw new ArgumentNullException(nameof(decoder)); + + if (decoder is IEnumerable composite) + { + return composite + .Select(nestedDecoder => nestedDecoder.GetDecoder(objectType, targetType)) + .FirstOrDefault(d => d != null); + } + + return decoder.CanDecode(objectType, targetType) ? decoder : null; + } + } +} diff --git a/src/runtime/Codecs/EncoderGroup.cs b/src/runtime/Codecs/EncoderGroup.cs new file mode 100644 index 000000000..4f776a669 --- /dev/null +++ b/src/runtime/Codecs/EncoderGroup.cs @@ -0,0 +1,76 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a group of s. Useful to group them by priority. + /// + public sealed class EncoderGroup: IPyObjectEncoder, IEnumerable + { + readonly List encoders = new List(); + + /// + /// Add specified encoder to the group + /// + public void Add(IPyObjectEncoder item) + { + if (item is null) throw new ArgumentNullException(nameof(item)); + this.encoders.Add(item); + } + /// + /// Remove all encoders from the group + /// + public void Clear() => this.encoders.Clear(); + + /// + public bool CanEncode(Type type) => this.encoders.Any(encoder => encoder.CanEncode(type)); + /// + public PyObject TryEncode(object value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + foreach (var encoder in this.GetEncoders(value.GetType())) + { + var result = encoder.TryEncode(value); + if (result != null) + { + return result; + } + } + + return null; + } + + /// + public IEnumerator GetEnumerator() => this.encoders.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => this.encoders.GetEnumerator(); + } + + public static class EncoderGroupExtensions + { + /// + /// Gets specific instances of + /// (potentially selecting one from a collection), + /// that can encode the specified . + /// + public static IEnumerable GetEncoders(this IPyObjectEncoder decoder, Type type) + { + if (decoder is null) throw new ArgumentNullException(nameof(decoder)); + + if (decoder is IEnumerable composite) + { + foreach (var nestedEncoder in composite) + foreach (var match in nestedEncoder.GetEncoders(type)) + { + yield return match; + } + } else if (decoder.CanEncode(type)) + { + yield return decoder; + } + } + } +} diff --git a/src/runtime/Codecs/RawProxyEncoder.cs b/src/runtime/Codecs/RawProxyEncoder.cs new file mode 100644 index 000000000..37ad0487b --- /dev/null +++ b/src/runtime/Codecs/RawProxyEncoder.cs @@ -0,0 +1,20 @@ +using System; + +namespace Python.Runtime.Codecs +{ + /// + /// A .NET object encoder, that returns raw proxies (e.g. no conversion to Python types). + /// You must inherit from this class and override . + /// + public class RawProxyEncoder: IPyObjectEncoder + { + public PyObject TryEncode(object value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return PyObject.FromManagedObject(value); + } + + public virtual bool CanEncode(Type type) => false; + } +} diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs new file mode 100644 index 000000000..5ac55846f --- /dev/null +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -0,0 +1,134 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder + { + TupleCodec() { } + public static TupleCodec Instance { get; } = new TupleCodec(); + + public bool CanEncode(Type type) + { + if (type == typeof(object) || type == typeof(TTuple)) return true; + return type.Namespace == typeof(TTuple).Namespace + // generic versions of tuples are named Tuple`TYPE_ARG_COUNT + && type.Name.StartsWith(typeof(TTuple).Name + '`'); + } + + public PyObject TryEncode(object value) + { + if (value == null) return null; + + var tupleType = value.GetType(); + if (tupleType == typeof(object)) return null; + if (!this.CanEncode(tupleType)) return null; + if (tupleType == typeof(TTuple)) return new PyTuple(); + + long fieldCount = tupleType.GetGenericArguments().Length; + var tuple = Runtime.PyTuple_New(fieldCount); + Exceptions.ErrorCheck(tuple); + int fieldIndex = 0; + foreach (FieldInfo field in tupleType.GetFields()) + { + var item = field.GetValue(value); + IntPtr pyItem = Converter.ToPython(item); + Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); + fieldIndex++; + } + return new PyTuple(tuple); + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == Runtime.PyTupleType && this.CanEncode(targetType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + value = default; + + if (!Runtime.PyTuple_Check(pyObj.Handle)) return false; + + if (typeof(T) == typeof(object)) + { + bool converted = Decode(pyObj, out object result); + if (converted) + { + value = (T)result; + return true; + } + + return false; + } + + var itemTypes = typeof(T).GetGenericArguments(); + long itemCount = Runtime.PyTuple_Size(pyObj.Handle); + if (itemTypes.Length != itemCount) return false; + + if (itemCount == 0) + { + value = (T)EmptyTuple; + return true; + } + + var elements = new object[itemCount]; + for (int itemIndex = 0; itemIndex < itemTypes.Length; itemIndex++) + { + IntPtr pyItem = Runtime.PyTuple_GetItem(pyObj.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, itemTypes[itemIndex], out elements[itemIndex], setError: false)) + { + Exceptions.Clear(); + return false; + } + } + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = (T)factory.Invoke(null, elements); + return true; + } + + static bool Decode(PyObject tuple, out object value) + { + long itemCount = Runtime.PyTuple_Size(tuple.Handle); + if (itemCount == 0) + { + value = EmptyTuple; + return true; + } + var elements = new object[itemCount]; + var itemTypes = new Type[itemCount]; + value = null; + for (int itemIndex = 0; itemIndex < elements.Length; itemIndex++) + { + var pyItem = Runtime.PyTuple_GetItem(tuple.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, typeof(object), out elements[itemIndex], setError: false)) + { + Exceptions.Clear(); + return false; + } + + itemTypes[itemIndex] = elements[itemIndex]?.GetType() ?? typeof(object); + } + + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = factory.Invoke(null, elements); + return true; + } + + static readonly MethodInfo[] tupleCreate = + typeof(TTuple).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Tuple.Create)) + .OrderBy(m => m.GetParameters().Length) + .ToArray(); + + static readonly object EmptyTuple = tupleCreate[0].Invoke(null, parameters: new object[0]); + + public static void Register() + { + PyObjectConversions.RegisterEncoder(Instance); + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index 90bb77a71..0cbbbaba2 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -91,13 +91,13 @@ public static int GetUnicodeByteLength(IntPtr p) var len = 0; while (true) { - int c = Runtime.UCS == 2 + int c = Runtime._UCS == 2 ? Marshal.ReadInt16(p, len * 2) : Marshal.ReadInt32(p, len * 4); if (c == 0) { - return len * Runtime.UCS; + return len * Runtime._UCS; } checked { @@ -120,9 +120,7 @@ public static int GetUnicodeByteLength(IntPtr p) /// public static IntPtr Py3UnicodePy2StringtoPtr(string s) { - return Runtime.IsPython3 - ? Instance.MarshalManagedToNative(s) - : Marshal.StringToHGlobalAnsi(s); + return Instance.MarshalManagedToNative(s); } /// @@ -137,9 +135,7 @@ public static IntPtr Py3UnicodePy2StringtoPtr(string s) /// public static string PtrToPy3UnicodePy2String(IntPtr p) { - return Runtime.IsPython3 - ? PtrToStringUni(p) - : Marshal.PtrToStringAnsi(p); + return PtrToStringUni(p); } } @@ -163,7 +159,7 @@ public override IntPtr MarshalManagedToNative(object managedObj) } int totalStrLength = argv.Sum(arg => arg.Length + 1); - int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime.UCS; + int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime._UCS; IntPtr mem = Marshal.AllocHGlobal(memSize); try diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs new file mode 100644 index 000000000..a4ed75918 --- /dev/null +++ b/src/runtime/NewReference.cs @@ -0,0 +1,82 @@ +namespace Python.Runtime +{ + using System; + using System.Diagnostics.Contracts; + + /// + /// Represents a reference to a Python object, that is tracked by Python's reference counting. + /// + [NonCopyable] + ref struct NewReference + { + IntPtr pointer; + + [Pure] + public static implicit operator BorrowedReference(in NewReference reference) + => new BorrowedReference(reference.pointer); + + /// + /// Returns wrapper around this reference, which now owns + /// the pointer. Sets the original reference to null, as it no longer owns it. + /// + public PyObject MoveToPyObject() + { + if (this.IsNull()) throw new NullReferenceException(); + + var result = new PyObject(this.pointer); + this.pointer = IntPtr.Zero; + return result; + } + + /// Moves ownership of this instance to unmanged pointer + public IntPtr DangerousMoveToPointerOrNull() + { + var result = this.pointer; + this.pointer = IntPtr.Zero; + return result; + } + + /// + /// Removes this reference to a Python object, and sets it to null. + /// + public void Dispose() + { + if (this.IsNull()) + { + return; + } + Runtime.XDecref(pointer); + pointer = IntPtr.Zero; + } + + /// + /// Creates from a raw pointer + /// + [Pure] + public static NewReference DangerousFromPointer(IntPtr pointer) + => new NewReference {pointer = pointer}; + + [Pure] + internal static IntPtr DangerousGetAddress(in NewReference reference) + => IsNull(reference) ? throw new NullReferenceException() : reference.pointer; + [Pure] + internal static bool IsNull(in NewReference reference) + => reference.pointer == IntPtr.Zero; + } + + /// + /// These members can not be directly in type, + /// because this is always passed by value, which we need to avoid. + /// (note this in NewReference vs the usual this NewReference) + /// + static class NewReferenceExtensions + { + /// Gets a raw pointer to the Python object + [Pure] + public static IntPtr DangerousGetAddress(this in NewReference reference) + => NewReference.DangerousGetAddress(reference); + [Pure] + public static bool IsNull(this in NewReference reference) + => NewReference.IsNull(reference); + } +} diff --git a/src/runtime/NonCopyableAttribute.cs b/src/runtime/NonCopyableAttribute.cs new file mode 100644 index 000000000..63d36ab42 --- /dev/null +++ b/src/runtime/NonCopyableAttribute.cs @@ -0,0 +1,6 @@ +namespace Python.Runtime +{ + using System; + [AttributeUsage(AttributeTargets.Struct)] + class NonCopyableAttribute : Attribute { } +} diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index a5d33c7ab..470488c02 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -1,11 +1,3 @@ -using System.Reflection; using System.Runtime.CompilerServices; -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Python for .NET")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyDefaultAlias("Python.Runtime.dll")] - [assembly: InternalsVisibleTo("Python.EmbeddingTest")] diff --git a/src/runtime/PyExportAttribute.cs b/src/runtime/PyExportAttribute.cs new file mode 100644 index 000000000..52a8be15d --- /dev/null +++ b/src/runtime/PyExportAttribute.cs @@ -0,0 +1,16 @@ +namespace Python.Runtime { + using System; + + /// + /// Controls visibility to Python for public .NET type or an entire assembly + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Delegate | AttributeTargets.Enum + | AttributeTargets.Interface | AttributeTargets.Struct | AttributeTargets.Assembly, + AllowMultiple = false, + Inherited = false)] + public class PyExportAttribute : Attribute + { + internal readonly bool Export; + public PyExportAttribute(bool export) { this.Export = export; } + } +} diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 8580b7f61..f18cf7a49 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,166 +1,33 @@ - - + - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime + netstandard2.0 + AnyCPU Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - false - ..\pythonnet.snk - - - x86 - - - x64 - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON36;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON36;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON36;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON36;UCS2;TRACE;DEBUG - false - full + Python.Runtime + pythonnet + https://github.com/pythonnet/pythonnet/blob/master/LICENSE + https://github.com/pythonnet/pythonnet + git + python interop dynamic dlr Mono pinvoke + https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico + https://pythonnet.github.io/ + 1591;NU1701 + True + + + + $(DefineConstants);$(ConfiguredConstants) + - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + clr.py - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + + diff --git a/src/runtime/ReferenceExtensions.cs b/src/runtime/ReferenceExtensions.cs new file mode 100644 index 000000000..8fa2731b7 --- /dev/null +++ b/src/runtime/ReferenceExtensions.cs @@ -0,0 +1,20 @@ +namespace Python.Runtime +{ + using System.Diagnostics.Contracts; + + static class ReferenceExtensions + { + /// + /// Checks if the reference points to Python object None. + /// + [Pure] + public static bool IsNone(this in NewReference reference) + => reference.DangerousGetAddress() == Runtime.PyNone; + /// + /// Checks if the reference points to Python object None. + /// + [Pure] + public static bool IsNone(this BorrowedReference reference) + => reference.DangerousGetAddress() == Runtime.PyNone; + } +} diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs new file mode 100644 index 000000000..e14e74bbc --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -0,0 +1,118 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T : MemberInfo + { + public static implicit operator MaybeMemberInfo(T ob) => new MaybeMemberInfo(ob); + + // .ToString() of the serialized object + const string SerializationName = "s"; + // The ReflectedType of the object + const string SerializationType = "t"; + const string SerializationFieldName = "f"; + string name; + MemberInfo info; + + [NonSerialized] + Exception deserializationException; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + } + } + + public T Value + { + get + { + if (info == null) + { + throw new SerializationException(DeletedMessage, innerException: deserializationException); + } + return (T)info; + } + } + + public string Name => name; + public bool Valid => info != null; + + public override string ToString() + { + return (info != null ? info.ToString() : $"missing type: {name}"); + } + + public MaybeMemberInfo(T fi) + { + info = fi; + name = info?.ToString(); + deserializationException = null; + } + + internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) + { + // Assumption: name is always stored in "s" + name = serializationInfo.GetString(SerializationName); + info = null; + deserializationException = null; + try + { + var tp = Type.GetType(serializationInfo.GetString(SerializationType)); + if (tp != null) + { + var field_name = serializationInfo.GetString(SerializationFieldName); + MemberInfo mi = tp.GetField(field_name, ClassManager.BindingFlags); + if (mi != null && ShouldBindMember(mi)) + { + info = mi; + } + } + } + catch (Exception e) + { + deserializationException = e; + } + } + + // This is complicated because we bind fields + // based on the visibility of the field, properties + // based on it's setter/getter (which is a method + // info) visibility and events based on their + // AddMethod visibility. + static bool ShouldBindMember(MemberInfo mi) + { + if (mi is PropertyInfo pi) + { + return ClassManager.ShouldBindProperty(pi); + } + else if (mi is FieldInfo fi) + { + return ClassManager.ShouldBindField(fi); + } + else if (mi is EventInfo ei) + { + return ClassManager.ShouldBindEvent(ei); + } + + return false; + } + + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + serializationInfo.AddValue(SerializationName, name); + if (Valid) + { + serializationInfo.AddValue(SerializationFieldName, info.Name); + serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); + } + } + } +} diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs new file mode 100644 index 000000000..d18c94059 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -0,0 +1,199 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Linq; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMethodBase : ISerializable where T: MethodBase + { + // .ToString() of the serialized object + const string SerializationName = "s"; + // The ReflectedType of the object + const string SerializationType = "t"; + // Fhe parameters of the MethodBase + const string SerializationParameters = "p"; + const string SerializationIsCtor = "c"; + const string SerializationMethodName = "n"; + + [Serializable] + struct ParameterHelper : IEquatable + { + public enum TypeModifier + { + None, + In, + Out, + Ref + } + public readonly string Name; + public readonly TypeModifier Modifier; + + public ParameterHelper(ParameterInfo tp) + { + Name = tp.ParameterType.AssemblyQualifiedName; + Modifier = TypeModifier.None; + + if (tp.IsIn && tp.ParameterType.IsByRef) + { + Modifier = TypeModifier.In; + } + else if (tp.IsOut && tp.ParameterType.IsByRef) + { + Modifier = TypeModifier.Out; + } + else if (tp.ParameterType.IsByRef) + { + Modifier = TypeModifier.Ref; + } + } + + public bool Equals(ParameterInfo other) + { + return this.Equals(new ParameterHelper(other)); + } + } + public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); + + string name; + MethodBase info; + + [NonSerialized] + Exception deserializationException; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + } + } + + public T Value + { + get + { + if (info == null) + { + throw new SerializationException(DeletedMessage, innerException: deserializationException); + } + return (T)info; + } + } + + public T UnsafeValue { get { return (T)info; } } + public string Name {get{return name;}} + public bool Valid => info != null; + + public override string ToString() + { + return (info != null ? info.ToString() : $"missing method info: {name}"); + } + + public MaybeMethodBase(T mi) + { + info = mi; + name = mi?.ToString(); + deserializationException = null; + } + + internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) + { + name = serializationInfo.GetString(SerializationName); + info = null; + deserializationException = null; + try + { + // Retrieve the reflected type of the method; + var typeName = serializationInfo.GetString(SerializationType); + var tp = Type.GetType(typeName); + if (tp == null) + { + throw new SerializationException($"The underlying type {typeName} can't be found"); + } + // Get the method's parameters types + var field_name = serializationInfo.GetString(SerializationMethodName); + var param = (ParameterHelper[])serializationInfo.GetValue(SerializationParameters, typeof(ParameterHelper[])); + Type[] types = new Type[param.Length]; + bool hasRefType = false; + for (int i = 0; i < param.Length; i++) + { + var paramTypeName = param[i].Name; + types[i] = Type.GetType(paramTypeName); + if (types[i] == null) + { + throw new SerializationException($"The parameter of type {paramTypeName} can't be found"); + } + else if (types[i].IsByRef) + { + hasRefType = true; + } + } + + MethodBase mb = null; + if (serializationInfo.GetBoolean(SerializationIsCtor)) + { + // We never want the static constructor. + mb = tp.GetConstructor(ClassManager.BindingFlags&(~BindingFlags.Static), binder:null, types:types, modifiers:null); + } + else + { + mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); + } + + if (mb != null && hasRefType) + { + mb = CheckRefTypes(mb, param); + } + + // Do like in ClassManager.GetClassInfo + if(mb != null && ClassManager.ShouldBindMethod(mb)) + { + info = mb; + } + } + catch (Exception e) + { + deserializationException = e; + } + } + + MethodBase CheckRefTypes(MethodBase mb, ParameterHelper[] ph) + { + // One more step: Changing: + // void MyFn (ref int a) + // to: + // void MyFn (out int a) + // will still find the function correctly as, `in`, `out` and `ref` + // are all represented as a reference type. Query the method we got + // and validate the parameters + if (ph.Length != 0) + { + foreach (var item in Enumerable.Zip(ph, mb.GetParameters(), (orig, current) => new {orig, current})) + { + if (!item.current.Equals(item.orig)) + { + // False positive + return null; + } + } + } + + return mb; + } + + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + serializationInfo.AddValue(SerializationName, name); + if (Valid) + { + serializationInfo.AddValue(SerializationMethodName, info.Name); + serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); + ParameterHelper[] parameters = (from p in info.GetParameters() select new ParameterHelper(p)).ToArray(); + serializationInfo.AddValue(SerializationParameters, parameters, typeof(ParameterHelper[])); + serializationInfo.AddValue(SerializationIsCtor, info.IsConstructor); + } + } + } +} \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs new file mode 100644 index 000000000..abb3a8fb6 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -0,0 +1,64 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeType : ISerializable + { + public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + + // The AssemblyQualifiedName of the serialized Type + const string SerializationName = "n"; + string name; + Type type; + + public string DeletedMessage + { + get + { + return $"The .NET Type {name} no longer exists"; + } + } + + public Type Value + { + get + { + if (type == null) + { + throw new SerializationException(DeletedMessage); + } + return type; + } + } + + public string Name => name; + public bool Valid => type != null; + + public override string ToString() + { + return (type != null ? type.ToString() : $"missing type: {name}"); + } + + public MaybeType(Type tp) + { + type = tp; + name = tp.AssemblyQualifiedName; + } + + private MaybeType(SerializationInfo serializationInfo, StreamingContext context) + { + name = (string)serializationInfo.GetValue(SerializationName, typeof(string)); + type = Type.GetType(name, throwOnError:false); + } + + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + serializationInfo.AddValue(SerializationName, name); + } + } +} \ No newline at end of file diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs new file mode 100644 index 000000000..eb21cddbb --- /dev/null +++ b/src/runtime/Util.cs @@ -0,0 +1,43 @@ +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime +{ + internal static class Util + { + internal const string UnstableApiMessage = + "This API is unstable, and might be changed or removed in the next minor release"; + + internal static Int64 ReadCLong(IntPtr tp, int offset) + { + // On Windows, a C long is always 32 bits. + if (Runtime.IsWindows || Runtime.Is32Bit) + { + return Marshal.ReadInt32(tp, offset); + } + else + { + return Marshal.ReadInt64(tp, offset); + } + } + + internal static void WriteCLong(IntPtr type, int offset, Int64 flags) + { + if (Runtime.IsWindows || Runtime.Is32Bit) + { + Marshal.WriteInt32(type, offset, (Int32)(flags & 0xffffffffL)); + } + else + { + Marshal.WriteInt64(type, offset, flags); + } + } + + /// + /// Null-coalesce: if parameter is not + /// , return it. Otherwise return . + /// + internal static IntPtr Coalesce(this IntPtr primary, IntPtr fallback) + => primary == IntPtr.Zero ? fallback : primary; + } +} diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index a10688749..262e521a5 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// the same as a ClassObject, except that it provides sequence semantics /// to support natural array usage (indexing) from Python. /// + [Serializable] internal class ArrayObject : ClassBase { internal ArrayObject(Type tp) : base(tp) @@ -19,32 +20,130 @@ internal override bool CanSubclass() return false; } - public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) + public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) { + if (kw != IntPtr.Zero) + { + return Exceptions.RaiseTypeError("array constructor takes no keyword arguments"); + } + + var tp = new BorrowedReference(tpRaw); + var self = GetManagedObject(tp) as ArrayObject; - if (Runtime.PyTuple_Size(args) != 1) + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type arrType = self.type.Value; + + long[] dimensions = new long[Runtime.PyTuple_Size(args)]; + if (dimensions.Length == 0) { - return Exceptions.RaiseTypeError("array expects 1 argument"); + return Exceptions.RaiseTypeError("array constructor requires at least one integer argument or an object convertible to array"); } + if (dimensions.Length != 1) + { + return CreateMultidimensional(arrType.GetElementType(), dimensions, + shapeTuple: new BorrowedReference(args), + pyType: tp) + .DangerousMoveToPointerOrNull(); + } + IntPtr op = Runtime.PyTuple_GetItem(args, 0); + + // create single dimensional array + if (Runtime.PyInt_Check(op)) + { + dimensions[0] = Runtime.PyLong_AsLongLong(op); + if (dimensions[0] == -1 && Exceptions.ErrorOccurred()) + { + Exceptions.Clear(); + } + else + { + return NewInstance(arrType.GetElementType(), tp, dimensions) + .DangerousMoveToPointerOrNull(); + } + } object result; - if (!Converter.ToManaged(op, self.type, out result, true)) + // this implements casting to Array[T] + if (!Converter.ToManaged(op, arrType, out result, true)) { return IntPtr.Zero; } - return CLRObject.GetInstHandle(result, tp); + return CLRObject.GetInstHandle(result, tp) + .DangerousGetAddress(); + } + + static NewReference CreateMultidimensional(Type elementType, long[] dimensions, BorrowedReference shapeTuple, BorrowedReference pyType) + { + for (int dimIndex = 0; dimIndex < dimensions.Length; dimIndex++) + { + BorrowedReference dimObj = Runtime.PyTuple_GetItem(shapeTuple, dimIndex); + PythonException.ThrowIfIsNull(dimObj); + + if (!Runtime.PyInt_Check(dimObj)) + { + Exceptions.RaiseTypeError("array constructor expects integer dimensions"); + return default; + } + + dimensions[dimIndex] = Runtime.PyLong_AsLongLong(dimObj); + if (dimensions[dimIndex] == -1 && Exceptions.ErrorOccurred()) + { + Exceptions.RaiseTypeError("array constructor expects integer dimensions"); + return default; + } + } + + return NewInstance(elementType, pyType, dimensions); + } + + static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, long[] dimensions) + { + object result; + try + { + result = Array.CreateInstance(elementType, dimensions); + } + catch (ArgumentException badArgument) + { + Exceptions.SetError(Exceptions.ValueError, badArgument.Message); + return default; + } + catch (OverflowException overflow) + { + Exceptions.SetError(overflow); + return default; + } + catch (NotSupportedException notSupported) + { + Exceptions.SetError(notSupported); + return default; + } + catch (OutOfMemoryException oom) + { + Exceptions.SetError(Exceptions.MemoryError, oom.Message); + return default; + } + return CLRObject.GetInstHandle(result, arrayPyType); } /// /// Implements __getitem__ for array types. /// - public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) + public new static IntPtr mp_subscript(IntPtr ob, IntPtr idx) { var obj = (CLRObject)GetManagedObject(ob); + var arrObj = (ArrayObject)GetManagedObjectType(ob); + if (!arrObj.type.Valid) + { + return Exceptions.RaiseTypeError(arrObj.type.DeletedMessage); + } var items = obj.inst as Array; - Type itemType = obj.inst.GetType().GetElementType(); + Type itemType = arrObj.type.Value.GetElementType(); int rank = items.Rank; int index; object value; @@ -93,7 +192,7 @@ public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) return IntPtr.Zero; } - int count = Runtime.PyTuple_Size(idx); + var count = Runtime.PyTuple_Size(idx); var args = new int[count]; @@ -132,7 +231,7 @@ public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) /// /// Implements __setitem__ for array types. /// - public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) + public static new int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) { var obj = (CLRObject)GetManagedObject(ob); var items = obj.inst as Array; @@ -186,7 +285,7 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) return -1; } - int count = Runtime.PyTuple_Size(idx); + var count = Runtime.PyTuple_Size(idx); var args = new int[count]; for (var i = 0; i < count; i++) @@ -244,16 +343,5 @@ public static int sq_contains(IntPtr ob, IntPtr v) return 0; } - - - /// - /// Implements __len__ for array types. - /// - public static int mp_length(IntPtr ob) - { - var self = (CLRObject)GetManagedObject(ob); - var items = self.inst as Array; - return items.Length; - } } } diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 06a4449a2..0387d2dfc 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Reflection; using System.Threading; @@ -17,16 +18,24 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - private static ConcurrentDictionary> namespaces; + // + // WARNING: Dangerous if cross-app domain usage is ever supported + // Reusing the dictionary with assemblies accross multiple initializations is problematic. + // Loading happens from CurrentDomain (see line 53). And if the first call is from AppDomain that is later unloaded, + // than it can end up referring to assemblies that are already unloaded (default behavior after unload appDomain - + // unless LoaderOptimization.MultiDomain is used); + // So for multidomain support it is better to have the dict. recreated for each app-domain initialization + private static ConcurrentDictionary> namespaces = + new ConcurrentDictionary>(); //private static Dictionary> generics; private static AssemblyLoadEventHandler lhandler; private static ResolveEventHandler rhandler; // updated only under GIL? - private static Dictionary probed; + private static Dictionary probed = new Dictionary(32); // modified from event handlers below, potentially triggered from different .NET threads - private static AssemblyList assemblies; + private static ConcurrentQueue assemblies; internal static List pypath; private AssemblyManager() @@ -40,10 +49,7 @@ private AssemblyManager() /// internal static void Initialize() { - namespaces = new ConcurrentDictionary>(); - probed = new Dictionary(32); - //generics = new Dictionary>(); - assemblies = new AssemblyList(16); + assemblies = new ConcurrentQueue(); pypath = new List(16); AppDomain domain = AppDomain.CurrentDomain; @@ -60,7 +66,7 @@ internal static void Initialize() try { ScanAssembly(a); - assemblies.Add(a); + assemblies.Enqueue(a); } catch (Exception ex) { @@ -91,7 +97,7 @@ internal static void Shutdown() private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) { Assembly assembly = args.LoadedAssembly; - assemblies.Add(assembly); + assemblies.Enqueue(assembly); ScanAssembly(assembly); } @@ -131,15 +137,15 @@ private static Assembly ResolveHandler(object ob, ResolveEventArgs args) /// internal static void UpdatePath() { - IntPtr list = Runtime.PySys_GetObject("path"); - int count = Runtime.PyList_Size(list); + BorrowedReference list = Runtime.PySys_GetObject("path"); + var count = Runtime.PyList_Size(list); if (count != pypath.Count) { pypath.Clear(); probed.Clear(); for (var i = 0; i < count; i++) { - IntPtr item = Runtime.PyList_GetItem(list, i); + BorrowedReference item = Runtime.PyList_GetItem(list, i); string path = Runtime.GetManagedString(item); if (path != null) { @@ -193,19 +199,14 @@ public static string FindAssembly(string name) /// public static Assembly LoadAssembly(string name) { - Assembly assembly = null; try { - assembly = Assembly.Load(name); + return Assembly.Load(name); } - catch (Exception) + catch (FileNotFoundException) { - //if (!(e is System.IO.FileNotFoundException)) - //{ - // throw; - //} + return null; } - return assembly; } @@ -215,18 +216,8 @@ public static Assembly LoadAssembly(string name) public static Assembly LoadAssemblyPath(string name) { string path = FindAssembly(name); - Assembly assembly = null; - if (path != null) - { - try - { - assembly = Assembly.LoadFrom(path); - } - catch (Exception) - { - } - } - return assembly; + if (path == null) return null; + return Assembly.LoadFrom(path); } /// @@ -236,25 +227,14 @@ public static Assembly LoadAssemblyPath(string name) /// public static Assembly LoadAssemblyFullPath(string name) { - Assembly assembly = null; if (Path.IsPathRooted(name)) { - if (!Path.HasExtension(name)) - { - name = name + ".dll"; - } if (File.Exists(name)) { - try - { - assembly = Assembly.LoadFrom(name); - } - catch (Exception) - { - } + return Assembly.LoadFrom(name); } } - return assembly; + return null; } /// @@ -272,66 +252,6 @@ public static Assembly FindLoadedAssembly(string name) return null; } - /// - /// Given a qualified name of the form A.B.C.D, attempt to load - /// an assembly named after each of A.B.C.D, A.B.C, A.B, A. This - /// will only actually probe for the assembly once for each unique - /// namespace. Returns true if any assemblies were loaded. - /// - /// - /// TODO item 3 "* Deprecate implicit loading of assemblies": - /// Set the fromFile flag if the name of the loaded assembly matches - /// the fully qualified name that was requested if the framework - /// actually loads an assembly. - /// Call ONLY for namespaces that HAVE NOT been cached yet. - /// - public static bool LoadImplicit(string name, bool warn = true) - { - string[] names = name.Split('.'); - var loaded = false; - var s = ""; - Assembly lastAssembly = null; - HashSet assembliesSet = null; - for (var i = 0; i < names.Length; i++) - { - s = i == 0 ? names[0] : s + "." + names[i]; - if (!probed.ContainsKey(s)) - { - if (assembliesSet == null) - { - assembliesSet = new HashSet(AppDomain.CurrentDomain.GetAssemblies()); - } - Assembly a = FindLoadedAssembly(s); - if (a == null) - { - a = LoadAssemblyPath(s); - } - if (a == null) - { - a = LoadAssembly(s); - } - if (a != null && !assembliesSet.Contains(a)) - { - loaded = true; - lastAssembly = a; - } - probed[s] = 1; - } - } - - // Deprecation warning - if (warn && loaded) - { - string location = Path.GetFileNameWithoutExtension(lastAssembly.Location); - string deprWarning = "The module was found, but not in a referenced namespace.\n" + - $"Implicit loading is deprecated. Please use clr.AddReference('{location}')."; - Exceptions.deprecation(deprWarning); - } - - return loaded; - } - - /// /// Scans an assembly for exported namespaces, adding them to the /// mapping of valid namespaces. Note that for a given namespace @@ -340,12 +260,14 @@ public static bool LoadImplicit(string name, bool warn = true) /// internal static void ScanAssembly(Assembly assembly) { + if (assembly.GetCustomAttribute()?.Export == false) + { + return; + } // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. - - Type[] types = assembly.GetTypes(); - foreach (Type t in types) + foreach (Type t in GetTypes(assembly)) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -419,10 +341,9 @@ public static List GetNames(string nsname) { foreach (Assembly a in namespaces[nsname].Keys) { - Type[] types = a.GetTypes(); - foreach (Type t in types) + foreach (Type t in GetTypes(a)) { - if ((t.Namespace ?? "") == nsname) + if ((t.Namespace ?? "") == nsname && !t.IsNested) { names.Add(t.Name); } @@ -445,119 +366,40 @@ public static List GetNames(string nsname) } /// - /// Returns the System.Type object for a given qualified name, + /// Returns the objects for the given qualified name, /// looking in the currently loaded assemblies for the named - /// type. Returns null if the named type cannot be found. + /// type. /// - public static Type LookupType(string qname) - { - foreach (Assembly assembly in assemblies) - { - Type type = assembly.GetType(qname); - if (type != null) - { - return type; - } - } - return null; - } + public static IEnumerable LookupTypes(string qualifiedName) + => assemblies.Select(assembly => assembly.GetType(qualifiedName)).Where(type => type != null && IsExported(type)); - /// - /// Wrapper around List<Assembly> for thread safe access - /// - private class AssemblyList : IEnumerable + internal static Type[] GetTypes(Assembly a) { - private readonly List _list; - private readonly ReaderWriterLockSlim _lock; - - public AssemblyList(int capacity) - { - _list = new List(capacity); - _lock = new ReaderWriterLockSlim(); - } - - public int Count - { - get - { - _lock.EnterReadLock(); - try - { - return _list.Count; - } - finally - { - _lock.ExitReadLock(); - } - } - } - - public void Add(Assembly assembly) + if (a.IsDynamic) { - _lock.EnterWriteLock(); try { - _list.Add(assembly); + return a.GetTypes().Where(IsExported).ToArray(); } - finally + catch (ReflectionTypeLoadException exc) { - _lock.ExitWriteLock(); + // Return all types that were successfully loaded + return exc.Types.Where(x => x != null && IsExported(x)).ToArray(); } } - - public IEnumerator GetEnumerator() + else { - return ((IEnumerable)this).GetEnumerator(); - } - - /// - /// Enumerator wrapping around 's enumerator. - /// Acquires and releases a read lock on during enumeration - /// - private class Enumerator : IEnumerator - { - private readonly AssemblyList _assemblyList; - - private readonly IEnumerator _listEnumerator; - - public Enumerator(AssemblyList assemblyList) - { - _assemblyList = assemblyList; - _assemblyList._lock.EnterReadLock(); - _listEnumerator = _assemblyList._list.GetEnumerator(); - } - - public void Dispose() - { - _listEnumerator.Dispose(); - _assemblyList._lock.ExitReadLock(); - } - - public bool MoveNext() - { - return _listEnumerator.MoveNext(); - } - - public void Reset() - { - _listEnumerator.Reset(); - } - - public Assembly Current + try { - get { return _listEnumerator.Current; } + return a.GetExportedTypes().Where(IsExported).ToArray(); } - - object IEnumerator.Current + catch (FileNotFoundException) { - get { return Current; } + return new Type[0]; } } - - IEnumerator IEnumerable.GetEnumerator() - { - return new Enumerator(this); - } } + + static bool IsExported(Type type) => type.GetCustomAttribute()?.Export != false; } } diff --git a/src/runtime/bufferinterface.cs b/src/runtime/bufferinterface.cs new file mode 100644 index 000000000..0c0ac2140 --- /dev/null +++ b/src/runtime/bufferinterface.cs @@ -0,0 +1,106 @@ +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime +{ + /* buffer interface */ + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] + internal struct Py_buffer { + public IntPtr buf; + public IntPtr obj; /* owned reference */ + [MarshalAs(UnmanagedType.SysInt)] + public IntPtr len; + [MarshalAs(UnmanagedType.SysInt)] + public IntPtr itemsize; /* This is Py_ssize_t so it can be + pointed to by strides in simple case.*/ + [MarshalAs(UnmanagedType.Bool)] + public bool _readonly; + public int ndim; + [MarshalAs(UnmanagedType.LPStr)] + public string format; + public IntPtr shape; + public IntPtr strides; + public IntPtr suboffsets; + public IntPtr _internal; + } + + public enum BufferOrderStyle + { + C, + Fortran, + EitherOne, + } + + /* Flags for getting buffers */ + public enum PyBUF + { + /// + /// Simple buffer without shape strides and suboffsets + /// + SIMPLE = 0, + /// + /// Controls the field. If set, the exporter MUST provide a writable buffer or else report failure. Otherwise, the exporter MAY provide either a read-only or writable buffer, but the choice MUST be consistent for all consumers. + /// + WRITABLE = 0x0001, + /// + /// Controls the field. If set, this field MUST be filled in correctly. Otherwise, this field MUST be NULL. + /// + FORMATS = 0x0004, + /// + /// N-Dimensional buffer with shape + /// + ND = 0x0008, + /// + /// Buffer with strides and shape + /// + STRIDES = (0x0010 | ND), + /// + /// C-Contigous buffer with strides and shape + /// + C_CONTIGUOUS = (0x0020 | STRIDES), + /// + /// F-Contigous buffer with strides and shape + /// + F_CONTIGUOUS = (0x0040 | STRIDES), + /// + /// C or Fortran contigous buffer with strides and shape + /// + ANY_CONTIGUOUS = (0x0080 | STRIDES), + /// + /// Buffer with suboffsets (if needed) + /// + INDIRECT = (0x0100 | STRIDES), + /// + /// Writable C-Contigous buffer with shape + /// + CONTIG = (ND | WRITABLE), + /// + /// Readonly C-Contigous buffer with shape + /// + CONTIG_RO = (ND), + /// + /// Writable buffer with shape and strides + /// + STRIDED = (STRIDES | WRITABLE), + /// + /// Readonly buffer with shape and strides + /// + STRIDED_RO = (STRIDES), + /// + /// Writable buffer with shape, strides and format + /// + RECORDS = (STRIDES | WRITABLE | FORMATS), + /// + /// Readonly buffer with shape, strides and format + /// + RECORDS_RO = (STRIDES | FORMATS), + /// + /// Writable indirect buffer with shape, strides, format and suboffsets (if needed) + /// + FULL = (INDIRECT | WRITABLE | FORMATS), + /// + /// Readonly indirect buffer with shape, strides, format and suboffsets (if needed) + /// + FULL_RO = (INDIRECT | FORMATS), + } +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 4dd3b5364..872501267 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,6 +1,9 @@ using System; using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -12,29 +15,36 @@ namespace Python.Runtime /// concrete subclasses provide slot implementations appropriate for /// each variety of reflected type. /// + [Serializable] internal class ClassBase : ManagedType { + [NonSerialized] + internal List dotNetMembers; internal Indexer indexer; - internal Type type; + internal Dictionary richcompare; + internal MaybeType type; internal ClassBase(Type tp) { + dotNetMembers = new List(); indexer = null; type = tp; } internal virtual bool CanSubclass() { - return !type.IsEnum; + return !type.Value.IsEnum; } - /// - /// Implements __init__ for reflected classes and value types. - /// - public static int tp_init(IntPtr ob, IntPtr args, IntPtr kw) + public readonly static Dictionary CilToPyOpMap = new Dictionary { - return 0; - } + ["op_Equality"] = Runtime.Py_EQ, + ["op_Inequality"] = Runtime.Py_NE, + ["op_LessThanOrEqual"] = Runtime.Py_LE, + ["op_GreaterThanOrEqual"] = Runtime.Py_GE, + ["op_LessThan"] = Runtime.Py_LT, + ["op_GreaterThan"] = Runtime.Py_GT, + }; /// /// Default implementation of [] semantics for reflected types. @@ -47,7 +57,12 @@ public virtual IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type target = GenericUtil.GenericForType(type, types.Length); + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + + Type target = GenericUtil.GenericForType(type.Value, types.Length); if (target != null) { @@ -57,7 +72,7 @@ public virtual IntPtr type_subscript(IntPtr idx) return c.pyHandle; } - return Exceptions.RaiseTypeError("no type matches params"); + return Exceptions.RaiseTypeError($"{type.Value.Namespace}.{type.Name} does not accept {types.Length} generic parameters"); } /// @@ -67,6 +82,30 @@ public static IntPtr tp_richcompare(IntPtr ob, IntPtr other, int op) { CLRObject co1; CLRObject co2; + IntPtr tp = Runtime.PyObject_TYPE(ob); + var cls = (ClassBase)GetManagedObject(tp); + // C# operator methods take precedence over IComparable. + // We first check if there's a comparison operator by looking up the richcompare table, + // otherwise fallback to checking if an IComparable interface is handled. + if (cls.richcompare.TryGetValue(op, out var methodObject)) + { + // Wrap the `other` argument of a binary comparison operator in a PyTuple. + IntPtr args = Runtime.PyTuple_New(1); + Runtime.XIncref(other); + Runtime.PyTuple_SetItem(args, 0, other); + + IntPtr value; + try + { + value = methodObject.Invoke(ob, args, IntPtr.Zero); + } + finally + { + Runtime.XDecref(args); // Free args pytuple + } + return value; + } + switch (op) { case Runtime.Py_EQ: @@ -188,7 +227,6 @@ public static IntPtr tp_iter(IntPtr ob) var e = co.inst as IEnumerable; IEnumerator o; - if (e != null) { o = e.GetEnumerator(); @@ -203,7 +241,22 @@ public static IntPtr tp_iter(IntPtr ob) } } - return new Iterator(o).pyHandle; + var elemType = typeof(object); + var iterType = co.inst.GetType(); + foreach(var ifc in iterType.GetInterfaces()) + { + if (ifc.IsGenericType) + { + var genTypeDef = ifc.GetGenericTypeDefinition(); + if (genTypeDef == typeof(IEnumerable<>) || genTypeDef == typeof(IEnumerator<>)) + { + elemType = ifc.GetGenericArguments()[0]; + break; + } + } + } + + return new Iterator(o, elemType).pyHandle; } @@ -246,40 +299,214 @@ public static IntPtr tp_str(IntPtr ob) } } + public static IntPtr tp_repr(IntPtr ob) + { + var co = GetManagedObject(ob) as CLRObject; + if (co == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + try + { + //if __repr__ is defined, use it + var instType = co.inst.GetType(); + System.Reflection.MethodInfo methodInfo = instType.GetMethod("__repr__"); + if (methodInfo != null && methodInfo.IsPublic) + { + var reprString = methodInfo.Invoke(co.inst, null) as string; + return Runtime.PyString_FromString(reprString); + } + + //otherwise use the standard object.__repr__(inst) + IntPtr args = Runtime.PyTuple_New(1); + Runtime.XIncref(ob); + Runtime.PyTuple_SetItem(args, 0, ob); + IntPtr reprFunc = Runtime.PyObject_GetAttr(Runtime.PyBaseObjectType, PyIdentifier.__repr__); + var output = Runtime.PyObject_Call(reprFunc, args, IntPtr.Zero); + Runtime.XDecref(args); + Runtime.XDecref(reprFunc); + return output; + } + catch (Exception e) + { + if (e.InnerException != null) + { + e = e.InnerException; + } + Exceptions.SetError(e); + return IntPtr.Zero; + } + } + /// - /// Default implementations for required Python GC support. + /// Standard dealloc implementation for instances of reflected types. /// - public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + public static void tp_dealloc(IntPtr ob) { - return 0; + ManagedType self = GetManagedObject(ob); + tp_clear(ob); + Runtime.PyObject_GC_UnTrack(self.pyHandle); + Runtime.PyObject_GC_Del(self.pyHandle); + self.FreeGCHandle(); } public static int tp_clear(IntPtr ob) { + ManagedType self = GetManagedObject(ob); + if (!self.IsTypeObject()) + { + ClearObjectDict(ob); + } + self.tpHandle = IntPtr.Zero; return 0; } - public static int tp_is_gc(IntPtr type) + protected override void OnSave(InterDomainContext context) { - return 1; + base.OnSave(context); + if (pyHandle != tpHandle) + { + IntPtr dict = GetObjectDict(pyHandle); + Runtime.XIncref(dict); + context.Storage.AddValue("dict", dict); + } } + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + if (pyHandle != tpHandle) + { + IntPtr dict = context.Storage.GetValue("dict"); + SetObjectDict(pyHandle, dict); + } + gcHandle = AllocGCHandle(); + Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + } + + /// - /// Standard dealloc implementation for instances of reflected types. + /// Implements __getitem__ for reflected classes and value types. /// - public static void tp_dealloc(IntPtr ob) + public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) { - ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); - if (dict != IntPtr.Zero) + IntPtr tp = Runtime.PyObject_TYPE(ob); + var cls = (ClassBase)GetManagedObject(tp); + + if (cls.indexer == null || !cls.indexer.CanGet) { - Runtime.XDecref(dict); + Exceptions.SetError(Exceptions.TypeError, "unindexable object"); + return IntPtr.Zero; } - Runtime.PyObject_GC_UnTrack(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); - self.gcHandle.Free(); + + // Arg may be a tuple in the case of an indexer with multiple + // parameters. If so, use it directly, else make a new tuple + // with the index arg (method binders expect arg tuples). + IntPtr args = idx; + var free = false; + + if (!Runtime.PyTuple_Check(idx)) + { + args = Runtime.PyTuple_New(1); + Runtime.XIncref(idx); + Runtime.PyTuple_SetItem(args, 0, idx); + free = true; + } + + IntPtr value; + + try + { + value = cls.indexer.GetItem(ob, args); + } + finally + { + if (free) + { + Runtime.XDecref(args); + } + } + return value; + } + + + /// + /// Implements __setitem__ for reflected classes and value types. + /// + public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) + { + IntPtr tp = Runtime.PyObject_TYPE(ob); + var cls = (ClassBase)GetManagedObject(tp); + + if (cls.indexer == null || !cls.indexer.CanSet) + { + Exceptions.SetError(Exceptions.TypeError, "object doesn't support item assignment"); + return -1; + } + + // Arg may be a tuple in the case of an indexer with multiple + // parameters. If so, use it directly, else make a new tuple + // with the index arg (method binders expect arg tuples). + IntPtr args = idx; + var free = false; + + if (!Runtime.PyTuple_Check(idx)) + { + args = Runtime.PyTuple_New(1); + Runtime.XIncref(idx); + Runtime.PyTuple_SetItem(args, 0, idx); + free = true; + } + + // Get the args passed in. + var i = Runtime.PyTuple_Size(args); + IntPtr defaultArgs = cls.indexer.GetDefaultArgs(args); + var numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); + var temp = i + numOfDefaultArgs; + IntPtr real = Runtime.PyTuple_New(temp + 1); + for (var n = 0; n < i; n++) + { + IntPtr item = Runtime.PyTuple_GetItem(args, n); + Runtime.XIncref(item); + Runtime.PyTuple_SetItem(real, n, item); + } + + // Add Default Args if needed + for (var n = 0; n < numOfDefaultArgs; n++) + { + IntPtr item = Runtime.PyTuple_GetItem(defaultArgs, n); + Runtime.XIncref(item); + Runtime.PyTuple_SetItem(real, n + i, item); + } + // no longer need defaultArgs + Runtime.XDecref(defaultArgs); + i = temp; + + // Add value to argument list + Runtime.XIncref(v); + Runtime.PyTuple_SetItem(real, i, v); + + try + { + cls.indexer.SetItem(ob, real); + } + finally + { + Runtime.XDecref(real); + + if (free) + { + Runtime.XDecref(args); + } + } + + if (Exceptions.ErrorOccurred()) + { + return -1; + } + + return 0; } } } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index c180f9acc..4e8e88bf3 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Reflection; using System.Reflection.Emit; +using System.Resources; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -21,6 +22,7 @@ public interface IPythonDerivedType { } + [Serializable] internal class ClassDerivedObject : ClassObject { private static Dictionary assemblyBuilders; @@ -32,6 +34,12 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } + public static void Reset() + { + assemblyBuilders = new Dictionary(); + moduleBuilders = new Dictionary, ModuleBuilder>(); + } + internal ClassDerivedObject(Type tp) : base(tp) { } @@ -67,7 +75,8 @@ internal ClassDerivedObject(Type tp) : base(tp) // So we don't call PyObject_GC_Del here and instead we set the python // reference to a weak reference so that the C# object can be collected. GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + int gcOffset = ObjectOffset.magic(Runtime.PyObject_TYPE(self.pyHandle)); + Marshal.WriteIntPtr(self.pyHandle, gcOffset, (IntPtr)gc); self.gcHandle.Free(); self.gcHandle = gc; } @@ -92,6 +101,10 @@ internal static IntPtr ToPython(IPythonDerivedType obj) // collected while Python still has a reference to it. if (Runtime.Refcount(self.pyHandle) == 1) { + +#if PYTHON_WITH_PYDEBUG + Runtime._Py_NewReference(self.pyHandle); +#endif GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); self.gcHandle.Free(); @@ -123,7 +136,7 @@ internal static Type CreateDerivedType(string name, if (null == assemblyName) { - assemblyName = Assembly.GetExecutingAssembly().FullName; + assemblyName = "Python.Runtime.Dynamic"; } ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); @@ -808,7 +821,6 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec obj, args); - var disposeList = new List(); CLRObject self = null; IntPtr gs = Runtime.PyGILState_Ensure(); try @@ -821,42 +833,9 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec // object to be collected. FieldInfo fi = obj.GetType().GetField("__pyobj__"); fi.SetValue(obj, self); - - Runtime.XIncref(self.pyHandle); - var pyself = new PyObject(self.pyHandle); - disposeList.Add(pyself); - - Runtime.XIncref(Runtime.PyNone); - var pynone = new PyObject(Runtime.PyNone); - disposeList.Add(pynone); - - // call __init__ - PyObject init = pyself.GetAttr("__init__", pynone); - disposeList.Add(init); - if (init.Handle != Runtime.PyNone) - { - // if __init__ hasn't been overridden then it will be a managed object - ManagedType managedMethod = ManagedType.GetManagedObject(init.Handle); - if (null == managedMethod) - { - var pyargs = new PyObject[args.Length]; - for (var i = 0; i < args.Length; ++i) - { - pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i]?.GetType())); - disposeList.Add(pyargs[i]); - } - - disposeList.Add(init.Invoke(pyargs)); - } - } } finally { - foreach (PyObject x in disposeList) - { - x?.Dispose(); - } - // Decrement the python object's reference count. // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. @@ -879,7 +858,7 @@ public static void Finalize(IPythonDerivedType obj) { if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) { - self.gcHandle.Free(); + if (self.gcHandle.IsAllocated) self.gcHandle.Free(); return; } } @@ -894,7 +873,7 @@ public static void Finalize(IPythonDerivedType obj) // If python's been terminated then just free the gchandle. if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) { - self.gcHandle.Free(); + if (self.gcHandle.IsAllocated) self.gcHandle.Free(); return; } @@ -904,7 +883,7 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 6a9d40ebd..0cbff371f 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -4,6 +4,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Security; +using System.Linq; namespace Python.Runtime { @@ -17,8 +18,21 @@ namespace Python.Runtime /// internal class ClassManager { - private static Dictionary cache; - private static Type dtype; + + // Binding flags to determine which members to expose in Python. + // This is complicated because inheritance in Python is name + // based. We can't just find DeclaredOnly members, because we + // could have a base class A that defines two overloads of a + // method and a class B that defines two more. The name-based + // descriptor Python will find needs to know about inherited + // overloads as well as those declared on the sub class. + internal static readonly BindingFlags BindingFlags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public | + BindingFlags.NonPublic; + + private static Dictionary cache; + private static readonly Type dtype; private ClassManager() { @@ -26,7 +40,6 @@ private ClassManager() static ClassManager() { - cache = new Dictionary(128); // SEE: https://msdn.microsoft.com/en-us/library/96b1ayy4(v=vs.100).aspx // ""All delegates inherit from MulticastDelegate, which inherits from Delegate."" // Was Delegate, which caused a null MethodInfo returned from GetMethode("Invoke") @@ -34,10 +47,134 @@ static ClassManager() dtype = typeof(MulticastDelegate); } + public static void Reset() + { + cache = new Dictionary(128); + } + + internal static void DisposePythonWrappersForClrTypes() + { + var visited = new HashSet(); + var visitedHandle = GCHandle.Alloc(visited); + var visitedPtr = (IntPtr)visitedHandle; + try + { + foreach (var cls in cache.Values) + { + // XXX: Force to release instance's managed resources + // but not dealloc itself immediately. + // These managed resources should preserve vacant shells + // since others may still referencing it. + cls.CallTypeTraverse(TraverseTypeClear, visitedPtr); + cls.CallTypeClear(); + cls.DecrRefCount(); + } + } + finally + { + visitedHandle.Free(); + } + cache.Clear(); + } + + private static int TraverseTypeClear(IntPtr ob, IntPtr arg) + { + var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; + if (!visited.Add(ob)) + { + return 0; + } + var clrObj = ManagedType.GetManagedObject(ob); + if (clrObj != null) + { + clrObj.CallTypeTraverse(TraverseTypeClear, arg); + clrObj.CallTypeClear(); + } + return 0; + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + var contexts = storage.AddValue("contexts", + new Dictionary()); + storage.AddValue("cache", cache); + foreach (var cls in cache) + { + if (!cls.Key.Valid) + { + // Don't serialize an invalid class + continue; + } + // This incref is for cache to hold the cls, + // thus no need for decreasing it at RestoreRuntimeData. + Runtime.XIncref(cls.Value.pyHandle); + var context = contexts[cls.Value.pyHandle] = new InterDomainContext(); + cls.Value.Save(context); + + // Remove all members added in InitBaseClass. + // this is done so that if domain reloads and a member of a + // reflected dotnet class is removed, it is removed from the + // Python object's dictionary tool; thus raising an AttributeError + // instead of a TypeError. + // Classes are re-initialized on in RestoreRuntimeData. + IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict); + foreach (var member in cls.Value.dotNetMembers) + { + // No need to decref the member, the ClassBase instance does + // not own the reference. + if ((Runtime.PyDict_DelItemString(dict, member) == -1) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } + } + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(cls.Value.tpHandle); + } + } + + internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) + { + cache = storage.GetValue>("cache"); + var invalidClasses = new List>(); + var contexts = storage.GetValue >("contexts"); + var loadedObjs = new Dictionary(); + foreach (var pair in cache) + { + if (!pair.Key.Valid) + { + invalidClasses.Add(pair); + continue; + } + // re-init the class + InitClassBase(pair.Key.Value, pair.Value); + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(pair.Value.tpHandle); + var context = contexts[pair.Value.pyHandle]; + pair.Value.Load(context); + loadedObjs.Add(pair.Value, context); + } + + foreach (var pair in invalidClasses) + { + cache.Remove(pair.Key); + Runtime.XDecref(pair.Value.pyHandle); + } + + return loadedObjs; + } + /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. /// + /// A Borrowed reference to the ClassBase object internal static ClassBase GetClass(Type type) { ClassBase cb = null; @@ -122,6 +259,7 @@ private static void InitClassBase(Type type, ClassBase impl) ClassInfo info = GetClassInfo(type); impl.indexer = info.indexer; + impl.richcompare = new Dictionary(); // Now we allocate the Python type object to reflect the given // managed type, filling the Python type slots with thunks that @@ -129,18 +267,27 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr tp = TypeManager.GetTypeHandle(impl, type); - impl.tpHandle = tp; // Finally, initialize the class __dict__ and return the object. IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); + if (impl.dotNetMembers == null) + { + impl.dotNetMembers = new List(); + } IDictionaryEnumerator iter = info.members.GetEnumerator(); while (iter.MoveNext()) { var item = (ManagedType)iter.Value; var name = (string)iter.Key; + impl.dotNetMembers.Add(name); Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + // Decref the item now that it's been used. + item.DecrRefCount(); + if (ClassBase.CilToPyOpMap.TryGetValue(name, out var pyOp)) { + impl.richcompare.Add(pyOp, (MethodObject)item); + } } // If class has constructors, generate an __doc__ attribute. @@ -156,7 +303,7 @@ private static void InitClassBase(Type type, ClassBase impl) var attr = (DocStringAttribute)attrs[0]; string docStr = attr.DocString; doc = Runtime.PyString_FromString(docStr); - Runtime.PyDict_SetItemString(dict, "__doc__", doc); + Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc); Runtime.XDecref(doc); } @@ -165,7 +312,7 @@ private static void InitClassBase(Type type, ClassBase impl) // required that the ClassObject.ctors be changed to internal if (co != null) { - if (co.ctors.Length > 0) + if (co.NumCtors > 0) { // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) @@ -173,24 +320,69 @@ private static void InitClassBase(Type type, ClassBase impl) var ctors = new ConstructorBinding(type, tp, co.binder); // ExtensionType types are untracked, so don't Incref() them. // TODO: deprecate __overloads__ soon... - Runtime.PyDict_SetItemString(dict, "__overloads__", ctors.pyHandle); - Runtime.PyDict_SetItemString(dict, "Overloads", ctors.pyHandle); + Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.pyHandle); + Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.pyHandle); + ctors.DecrRefCount(); } // don't generate the docstring if one was already set from a DocStringAttribute. if (!CLRModule._SuppressDocs && doc == IntPtr.Zero) { doc = co.GetDocString(); - Runtime.PyDict_SetItemString(dict, "__doc__", doc); + Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc); Runtime.XDecref(doc); } } } + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(tp); + } + + internal static bool ShouldBindMethod(MethodBase mb) + { + return (mb.IsPublic || mb.IsFamily || mb.IsFamilyOrAssembly); + } + + internal static bool ShouldBindField(FieldInfo fi) + { + return (fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly); + } + + internal static bool ShouldBindProperty(PropertyInfo pi) + { + MethodInfo mm = null; + try + { + mm = pi.GetGetMethod(true); + if (mm == null) + { + mm = pi.GetSetMethod(true); + } + } + catch (SecurityException) + { + // GetGetMethod may try to get a method protected by + // StrongNameIdentityPermission - effectively private. + return false; + } + + if (mm == null) + { + return false; + } + + return ShouldBindMethod(mm); + } + + internal static bool ShouldBindEvent(EventInfo ei) + { + return ShouldBindMethod(ei.GetAddMethod(true)); } private static ClassInfo GetClassInfo(Type type) { - var ci = new ClassInfo(type); + var ci = new ClassInfo(); var methods = new Hashtable(); ArrayList list; MethodInfo meth; @@ -200,18 +392,7 @@ private static ClassInfo GetClassInfo(Type type) Type tp; int i, n; - // This is complicated because inheritance in Python is name - // based. We can't just find DeclaredOnly members, because we - // could have a base class A that defines two overloads of a - // method and a class B that defines two more. The name-based - // descriptor Python will find needs to know about inherited - // overloads as well as those declared on the sub class. - BindingFlags flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public | - BindingFlags.NonPublic; - - MemberInfo[] info = type.GetMembers(flags); + MemberInfo[] info = type.GetMembers(BindingFlags); var local = new Hashtable(); var items = new ArrayList(); MemberInfo m; @@ -254,7 +435,7 @@ private static ClassInfo GetClassInfo(Type type) for (i = 0; i < inheritedInterfaces.Length; ++i) { Type inheritedType = inheritedInterfaces[i]; - MemberInfo[] imembers = inheritedType.GetMembers(flags); + MemberInfo[] imembers = inheritedType.GetMembers(BindingFlags); for (n = 0; n < imembers.Length; n++) { m = imembers[n]; @@ -264,6 +445,17 @@ private static ClassInfo GetClassInfo(Type type) } } } + + // All interface implementations inherit from Object, + // but GetMembers don't return them either. + var objFlags = BindingFlags.Public | BindingFlags.Instance; + foreach (var mi in typeof(object).GetMembers(objFlags)) + { + if (local[mi.Name] == null) + { + items.Add(mi); + } + } } for (i = 0; i < items.Count; i++) @@ -274,8 +466,7 @@ private static ClassInfo GetClassInfo(Type type) { case MemberTypes.Method: meth = (MethodInfo)mi; - if (!(meth.IsPublic || meth.IsFamily || - meth.IsFamilyOrAssembly)) + if (!ShouldBindMethod(meth)) { continue; } @@ -292,28 +483,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Property: var pi = (PropertyInfo)mi; - MethodInfo mm = null; - try - { - mm = pi.GetGetMethod(true); - if (mm == null) - { - mm = pi.GetSetMethod(true); - } - } - catch (SecurityException) - { - // GetGetMethod may try to get a method protected by - // StrongNameIdentityPermission - effectively private. - continue; - } - - if (mm == null) - { - continue; - } - - if (!(mm.IsPublic || mm.IsFamily || mm.IsFamilyOrAssembly)) + if(!ShouldBindProperty(pi)) { continue; } @@ -338,7 +508,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Field: var fi = (FieldInfo)mi; - if (!(fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly)) + if (!ShouldBindField(fi)) { continue; } @@ -348,8 +518,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Event: var ei = (EventInfo)mi; - MethodInfo me = ei.GetAddMethod(true); - if (!(me.IsPublic || me.IsFamily || me.IsFamilyOrAssembly)) + if (!ShouldBindEvent(ei)) { continue; } @@ -366,6 +535,8 @@ private static ClassInfo GetClassInfo(Type type) } // Note the given instance might be uninitialized ob = GetClass(tp); + // GetClass returns a Borrowed ref. ci.members owns the reference. + ob.IncrRefCount(); ci.members[mi.Name] = ob; continue; } @@ -382,22 +553,57 @@ private static ClassInfo GetClassInfo(Type type) ob = new MethodObject(type, name, mlist); ci.members[name] = ob; + if (mlist.Any(OperatorMethod.IsOperatorMethod)) + { + string pyName = OperatorMethod.GetPyMethodName(name); + string pyNameReverse = OperatorMethod.ReversePyMethodName(pyName); + OperatorMethod.FilterMethods(mlist, out var forwardMethods, out var reverseMethods); + // Only methods where the left operand is the declaring type. + if (forwardMethods.Length > 0) + ci.members[pyName] = new MethodObject(type, name, forwardMethods); + // Only methods where only the right operand is the declaring type. + if (reverseMethods.Length > 0) + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods); + } + } + + if (ci.indexer == null && type.IsClass) + { + // Indexer may be inherited. + var parent = type.BaseType; + while (parent != null && ci.indexer == null) + { + foreach (var prop in parent.GetProperties()) { + var args = prop.GetIndexParameters(); + if (args.GetLength(0) > 0) + { + ci.indexer = new Indexer(); + ci.indexer.AddProperty(prop); + break; + } + } + parent = parent.BaseType; + } } return ci; } - } - - - internal class ClassInfo - { - public Indexer indexer; - public Hashtable members; - - internal ClassInfo(Type t) + + /// + /// This class owns references to PyObjects in the `members` member. + /// The caller has responsibility to DECREF them. + /// + private class ClassInfo { - members = new Hashtable(); - indexer = null; + public Indexer indexer; + public Hashtable members; + + internal ClassInfo() + { + members = new Hashtable(); + indexer = null; + } } } + } diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 46257c73f..826ae5c54 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -1,3 +1,4 @@ +using System.Linq; using System; using System.Reflection; @@ -9,17 +10,18 @@ namespace Python.Runtime /// Python type objects. Each of those type objects is associated with /// an instance of ClassObject, which provides its implementation. /// + [Serializable] internal class ClassObject : ClassBase { internal ConstructorBinder binder; - internal ConstructorInfo[] ctors; + internal int NumCtors = 0; internal ClassObject(Type tp) : base(tp) { - ctors = type.GetConstructors(); - binder = new ConstructorBinder(type); - - foreach (ConstructorInfo t in ctors) + var _ctors = type.Value.GetConstructors(); + NumCtors = _ctors.Length; + binder = new ConstructorBinder(type.Value); + foreach (ConstructorInfo t in _ctors) { binder.AddMethod(t); } @@ -60,7 +62,11 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("invalid object"); } - Type type = self.type; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type type = self.type.Value; // Primitive types do not have constructors, but they look like // they do from Python. If the ClassObject represents one of the @@ -113,16 +119,21 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) /// public override IntPtr type_subscript(IntPtr idx) { + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + // If this type is the Array type, the [] means we need to // construct and return an array type of the given element type. - if (type == typeof(Array)) + if (type.Value == typeof(Array)) { if (Runtime.PyTuple_Check(idx)) { return Exceptions.RaiseTypeError("type expected"); } var c = GetManagedObject(idx) as ClassBase; - Type t = c != null ? c.type : Converter.GetTypeByAlias(idx); + Type t = c != null ? c.type.Value : Converter.GetTypeByAlias(idx); if (t == null) { return Exceptions.RaiseTypeError("type expected"); @@ -142,7 +153,7 @@ public override IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type gtype = AssemblyManager.LookupType($"{type.FullName}`{types.Length}"); + Type gtype = AssemblyManager.LookupTypes($"{type.Value.FullName}`{types.Length}").FirstOrDefault(); if (gtype != null) { var g = ClassManager.GetClass(gtype) as GenericType; @@ -152,162 +163,5 @@ public override IntPtr type_subscript(IntPtr idx) } return Exceptions.RaiseTypeError("unsubscriptable object"); } - - - /// - /// Implements __getitem__ for reflected classes and value types. - /// - public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) - { - //ManagedType self = GetManagedObject(ob); - IntPtr tp = Runtime.PyObject_TYPE(ob); - var cls = (ClassBase)GetManagedObject(tp); - - if (cls.indexer == null || !cls.indexer.CanGet) - { - Exceptions.SetError(Exceptions.TypeError, "unindexable object"); - return IntPtr.Zero; - } - - // Arg may be a tuple in the case of an indexer with multiple - // parameters. If so, use it directly, else make a new tuple - // with the index arg (method binders expect arg tuples). - IntPtr args = idx; - var free = false; - - if (!Runtime.PyTuple_Check(idx)) - { - args = Runtime.PyTuple_New(1); - Runtime.XIncref(idx); - Runtime.PyTuple_SetItem(args, 0, idx); - free = true; - } - - IntPtr value; - - try - { - value = cls.indexer.GetItem(ob, args); - } - finally - { - if (free) - { - Runtime.XDecref(args); - } - } - return value; - } - - - /// - /// Implements __setitem__ for reflected classes and value types. - /// - public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) - { - //ManagedType self = GetManagedObject(ob); - IntPtr tp = Runtime.PyObject_TYPE(ob); - var cls = (ClassBase)GetManagedObject(tp); - - if (cls.indexer == null || !cls.indexer.CanSet) - { - Exceptions.SetError(Exceptions.TypeError, "object doesn't support item assignment"); - return -1; - } - - // Arg may be a tuple in the case of an indexer with multiple - // parameters. If so, use it directly, else make a new tuple - // with the index arg (method binders expect arg tuples). - IntPtr args = idx; - var free = false; - - if (!Runtime.PyTuple_Check(idx)) - { - args = Runtime.PyTuple_New(1); - Runtime.XIncref(idx); - Runtime.PyTuple_SetItem(args, 0, idx); - free = true; - } - - // Get the args passed in. - int i = Runtime.PyTuple_Size(args); - IntPtr defaultArgs = cls.indexer.GetDefaultArgs(args); - int numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); - int temp = i + numOfDefaultArgs; - IntPtr real = Runtime.PyTuple_New(temp + 1); - for (var n = 0; n < i; n++) - { - IntPtr item = Runtime.PyTuple_GetItem(args, n); - Runtime.XIncref(item); - Runtime.PyTuple_SetItem(real, n, item); - } - - // Add Default Args if needed - for (var n = 0; n < numOfDefaultArgs; n++) - { - IntPtr item = Runtime.PyTuple_GetItem(defaultArgs, n); - Runtime.XIncref(item); - Runtime.PyTuple_SetItem(real, n + i, item); - } - // no longer need defaultArgs - Runtime.XDecref(defaultArgs); - i = temp; - - // Add value to argument list - Runtime.XIncref(v); - Runtime.PyTuple_SetItem(real, i, v); - - try - { - cls.indexer.SetItem(ob, real); - } - finally - { - Runtime.XDecref(real); - - if (free) - { - Runtime.XDecref(args); - } - } - - if (Exceptions.ErrorOccurred()) - { - return -1; - } - - return 0; - } - - - /// - /// This is a hack. Generally, no managed class is considered callable - /// from Python - with the exception of System.Delegate. It is useful - /// to be able to call a System.Delegate instance directly, especially - /// when working with multicast delegates. - /// - public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) - { - //ManagedType self = GetManagedObject(ob); - IntPtr tp = Runtime.PyObject_TYPE(ob); - var cb = (ClassBase)GetManagedObject(tp); - - if (cb.type != typeof(Delegate)) - { - Exceptions.SetError(Exceptions.TypeError, "object is not callable"); - return IntPtr.Zero; - } - - var co = (CLRObject)GetManagedObject(ob); - var d = co.inst as Delegate; - BindingFlags flags = BindingFlags.Public | - BindingFlags.NonPublic | - BindingFlags.Instance | - BindingFlags.Static; - - MethodInfo method = d.GetType().GetMethod("Invoke", flags); - var binder = new MethodBinder(method); - return binder.Invoke(ob, args, kw); - } } } diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 472e5dcbb..0a352b381 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,32 +1,34 @@ using System; +using System.Diagnostics; using System.Runtime.InteropServices; namespace Python.Runtime { + [Serializable] internal class CLRObject : ManagedType { internal object inst; internal CLRObject(object ob, IntPtr tp) { + System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - var flags = (int)Marshal.ReadIntPtr(tp, TypeOffset.tp_flags); + long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.DictOffset(tp)); + IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); if (dict == IntPtr.Zero) { dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.DictOffset(tp), dict); + Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); } } - GCHandle gc = GCHandle.Alloc(this); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; - gcHandle = gc; inst = ob; // Fix the BaseException args (and __cause__ in case of Python 3) @@ -34,20 +36,27 @@ internal CLRObject(object ob, IntPtr tp) Exceptions.SetArgsAndCause(py); } + protected CLRObject() + { + } - internal static CLRObject GetInstance(object ob, IntPtr pyType) + static CLRObject GetInstance(object ob, IntPtr pyType) { return new CLRObject(ob, pyType); } - internal static CLRObject GetInstance(object ob) + static CLRObject GetInstance(object ob) { ClassBase cc = ClassManager.GetClass(ob.GetType()); return GetInstance(ob, cc.tpHandle); } - + internal static NewReference GetInstHandle(object ob, BorrowedReference pyType) + { + CLRObject co = GetInstance(ob, pyType.DangerousGetAddress()); + return NewReference.DangerousFromPointer(co.pyHandle); + } internal static IntPtr GetInstHandle(object ob, IntPtr pyType) { CLRObject co = GetInstance(ob, pyType); @@ -68,5 +77,31 @@ internal static IntPtr GetInstHandle(object ob) CLRObject co = GetInstance(ob); return co.pyHandle; } + + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) + { + CLRObject co = new CLRObject() + { + inst = ob, + pyHandle = pyHandle, + tpHandle = Runtime.PyObject_TYPE(pyHandle) + }; + Debug.Assert(co.tpHandle != IntPtr.Zero); + co.Load(context); + return co; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + Runtime.XIncref(pyHandle); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + } } } diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 1fc541920..83f2c81e4 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Text; namespace Python.Runtime { @@ -10,9 +11,10 @@ namespace Python.Runtime /// standard MethodBinder because of a difference in invoking constructors /// using reflection (which is seems to be a CLR bug). /// + [Serializable] internal class ConstructorBinder : MethodBinder { - private Type _containingType; + private MaybeType _containingType; internal ConstructorBinder(Type containingType) { @@ -49,10 +51,15 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw) /// internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) { + if (!_containingType.Valid) + { + return Exceptions.RaiseTypeError(_containingType.DeletedMessage); + } object result; + Type tp = _containingType.Value; - if (_containingType.IsValueType && !_containingType.IsPrimitive && - !_containingType.IsEnum && _containingType != typeof(decimal) && + if (tp.IsValueType && !tp.IsPrimitive && + !tp.IsEnum && tp != typeof(decimal) && Runtime.PyTuple_Size(args) == 0) { // If you are trying to construct an instance of a struct by @@ -62,7 +69,7 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) // Activator.CreateInstance(). try { - result = Activator.CreateInstance(_containingType); + result = Activator.CreateInstance(tp); } catch (Exception e) { @@ -88,12 +95,20 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) // any extra args are intended for the subclass' __init__. IntPtr eargs = Runtime.PyTuple_New(0); - binding = Bind(inst, eargs, kw); + binding = Bind(inst, eargs, IntPtr.Zero); Runtime.XDecref(eargs); if (binding == null) { - Exceptions.SetError(Exceptions.TypeError, "no constructor matches given arguments"); + var errorMessage = new StringBuilder("No constructor matches given arguments"); + if (info != null && info.IsConstructor && info.DeclaringType != null) + { + errorMessage.Append(" for ").Append(info.DeclaringType.Name); + } + + errorMessage.Append(": "); + AppendArgumentTypes(to: errorMessage, args); + Exceptions.SetError(Exceptions.TypeError, errorMessage.ToString()); return null; } } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 4839f9913..803823e39 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -19,18 +19,20 @@ namespace Python.Runtime /// and creating the BoundContructor object which contains ContructorInfo object. /// 3) In tp_call, if ctorInfo is not null, ctorBinder.InvokeRaw() is called. /// + [Serializable] internal class ConstructorBinding : ExtensionType { - private Type type; // The managed Type being wrapped in a ClassObject + private MaybeType type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; + + [NonSerialized] private IntPtr repr; public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; repr = IntPtr.Zero; } @@ -75,7 +77,7 @@ public static IntPtr tp_descr_get(IntPtr op, IntPtr instance, IntPtr owner) return Exceptions.RaiseTypeError("How in the world could that happen!"); } }*/ - Runtime.XIncref(self.pyHandle); // Decref'd by the interpreter. + Runtime.XIncref(self.pyHandle); return self.pyHandle; } @@ -90,6 +92,11 @@ public static IntPtr tp_descr_get(IntPtr op, IntPtr instance, IntPtr owner) public static IntPtr mp_subscript(IntPtr op, IntPtr key) { var self = (ConstructorBinding)GetManagedObject(op); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type tp = self.type.Value; Type[] types = Runtime.PythonArgsToTypeArray(key); if (types == null) @@ -98,15 +105,13 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) } //MethodBase[] methBaseArray = self.ctorBinder.GetMethods(); //MethodBase ci = MatchSignature(methBaseArray, types); - ConstructorInfo ci = self.type.GetConstructor(types); + ConstructorInfo ci = tp.GetConstructor(types); if (ci == null) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(self.type, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(tp, self.pyTypeHndl, self.ctorBinder, ci); - /* Since nothing is cached, do we need the increment??? - Runtime.XIncref(boundCtor.pyHandle); // Decref'd by the interpreter??? */ return boundCtor.pyHandle; } @@ -122,7 +127,12 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } MethodBase[] methods = self.ctorBinder.GetMethods(); - string name = self.type.FullName; + + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + string name = self.type.Value.FullName; var doc = ""; foreach (MethodBase t in methods) { @@ -146,8 +156,25 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (ConstructorBinding)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); - ExtensionType.FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (ConstructorBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ConstructorBinding)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; } } @@ -159,6 +186,7 @@ public static IntPtr tp_repr(IntPtr ob) /// An earlier implementation hung the __call__ on the ContructorBinding class and /// returned an Incref()ed self.pyHandle from the __get__ function. /// + [Serializable] internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject @@ -170,8 +198,7 @@ internal class BoundContructor : ExtensionType public BoundContructor(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; ctorInfo = ci; repr = IntPtr.Zero; @@ -232,8 +259,25 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (BoundContructor)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); - FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (BoundContructor)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (BoundContructor)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; } } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index b66150a29..e1b689cf3 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,7 +1,8 @@ using System; using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; @@ -84,9 +85,6 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == int32Type) return Runtime.PyIntType; - if (op == int64Type && Runtime.IsPython2) - return Runtime.PyLongType; - if (op == int64Type) return Runtime.PyIntType; @@ -114,6 +112,23 @@ internal static IntPtr ToPython(T value) return ToPython(value, typeof(T)); } + private static readonly Func IsTransparentProxy = GetIsTransparentProxy(); + + private static bool Never(object _) => false; + + private static Func GetIsTransparentProxy() + { + var remoting = typeof(int).Assembly.GetType("System.Runtime.Remoting.RemotingServices"); + if (remoting is null) return Never; + + var isProxy = remoting.GetMethod("IsTransparentProxy", new[] { typeof(object) }); + if (isProxy is null) return Never; + + return (Func)Delegate.CreateDelegate( + typeof(Func), isProxy, + throwOnBindFailure: true); + } + internal static IntPtr ToPython(object value, Type type) { if (value is PyObject) @@ -133,12 +148,53 @@ internal static IntPtr ToPython(object value, Type type) return result; } + if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { + var encoded = PyObjectConversions.TryEncode(value, type); + if (encoded != null) { + result = encoded.Handle; + Runtime.XIncref(result); + return result; + } + } + + if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) + { + using (var resultlist = new PyList()) + { + foreach (object o in (IEnumerable)value) + { + using (var p = new PyObject(ToPython(o, o?.GetType()))) + { + resultlist.Append(p); + } + } + Runtime.XIncref(resultlist.Handle); + return resultlist.Handle; + } + } + + if (type.IsInterface) + { + var ifaceObj = (InterfaceObject)ClassManager.GetClass(type); + return ifaceObj.WrapObject(value); + } + + // We need to special case interface array handling to ensure we + // produce the correct type. Value may be an array of some concrete + // type (FooImpl[]), but we want access to go via the interface type + // (IFoo[]). + if (type.IsArray && type.GetElementType().IsInterface) + { + return CLRObject.GetInstHandle(value, type); + } + // it the type is a python subclass of a managed type then return the // underlying python object rather than construct a new wrapper object. var pyderived = value as IPythonDerivedType; if (null != pyderived) { - return ClassDerivedObject.ToPython(pyderived); + if (!IsTransparentProxy(pyderived)) + return ClassDerivedObject.ToPython(pyderived); } // hmm - from Python, we almost never care what the declared @@ -257,7 +313,9 @@ internal static bool ToManaged(IntPtr value, Type type, return Converter.ToManagedValue(value, type, out result, setError); } - + internal static bool ToManagedValue(BorrowedReference value, Type obType, + out object result, bool setError) + => ToManagedValue(value.DangerousGetAddress(), obType, out result, setError); internal static bool ToManagedValue(IntPtr value, Type obType, out object result, bool setError) { @@ -288,7 +346,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (mt is ClassBase) { - result = ((ClassBase)mt).type; + var cb = (ClassBase)mt; + if (!cb.type.Valid) + { + Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); + return false; + } + result = cb.type.Value; return true; } // shouldn't happen @@ -301,6 +365,17 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return true; } + if (obType.IsGenericType && obType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + if( value == Runtime.PyNone ) + { + result = null; + return true; + } + // Set type to underlying type + obType = obType.GetGenericArguments()[0]; + } + if (obType.IsArray) { return ToArray(value, obType, out result, setError); @@ -340,17 +415,21 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToPrimitive(value, doubleType, out result, setError); } - if (Runtime.PySequence_Check(value)) + // give custom codecs a chance to take over conversion of sequences + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) { - return ToArray(value, typeof(object[]), out result, setError); + return true; } - if (setError) + if (Runtime.PySequence_Check(value)) { - Exceptions.SetError(Exceptions.TypeError, "value cannot be converted to Object"); + return ToArray(value, typeof(object[]), out result, setError); } - return false; + Runtime.XIncref(value); // PyObject() assumes ownership + result = new PyObject(value); + return true; } // Conversion to 'Type' is done using the same mappings as above for objects. @@ -400,19 +479,29 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + TypeCode typeCode = Type.GetTypeCode(obType); + if (typeCode == TypeCode.Object) + { + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + } + return ToPrimitive(value, obType, out result, setError); } + internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + /// /// Convert a Python value to an instance of a primitive managed type. /// private static bool ToPrimitive(IntPtr value, Type obType, out object result, bool setError) { - IntPtr overflow = Exceptions.OverflowError; TypeCode tc = Type.GetTypeCode(obType); result = null; - IntPtr op; - int ival; + IntPtr op = IntPtr.Zero; switch (tc) { @@ -426,54 +515,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo return true; case TypeCode.Int32: - // Trickery to support 64-bit platforms. - if (Runtime.IsPython2 && Runtime.Is32Bit) { - op = Runtime.PyNumber_Int(value); - - // As of Python 2.3, large ints magically convert :( - if (Runtime.PyLong_Check(op)) + // Python3 always use PyLong API + long num = Runtime.PyLong_AsLongLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) { - Runtime.XDecref(op); - goto overflow; + goto convert_error; } - - if (op == IntPtr.Zero) - { - if (Exceptions.ExceptionMatches(overflow)) - { - goto overflow; - } - goto type_error; - } - ival = (int)Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - result = ival; - return true; - } - else // Python3 always use PyLong API - { - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) - { - Exceptions.Clear(); - if (Exceptions.ExceptionMatches(overflow)) - { - goto overflow; - } - goto type_error; - } - long ll = (long)Runtime.PyLong_AsLongLong(op); - Runtime.XDecref(op); - if (ll == -1 && Exceptions.ErrorOccurred()) + if (num > Int32.MaxValue || num < Int32.MinValue) { goto overflow; } - if (ll > Int32.MaxValue || ll < Int32.MinValue) - { - goto overflow; - } - result = (int)ll; + result = (int)num; return true; } @@ -482,308 +535,258 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo return true; case TypeCode.Byte: -#if PYTHON3 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { - if (Runtime.PyBytes_Size(value) == 1) + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { - op = Runtime.PyBytes_AS_STRING(value); - result = (byte)Marshal.ReadByte(op); - return true; + if (Runtime.PyBytes_Size(value) == 1) + { + op = Runtime.PyBytes_AS_STRING(value); + result = (byte)Marshal.ReadByte(op); + return true; + } + goto type_error; } - goto type_error; - } -#elif PYTHON2 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) - { - if (Runtime.PyString_Size(value) == 1) + + int num = Runtime.PyLong_AsLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) { - op = Runtime.PyString_AsString(value); - result = (byte)Marshal.ReadByte(op); - return true; + goto convert_error; } - goto type_error; - } -#endif - - op = Runtime.PyNumber_Int(value); - if (op == IntPtr.Zero) - { - if (Exceptions.ExceptionMatches(overflow)) + if (num > Byte.MaxValue || num < Byte.MinValue) { goto overflow; } - goto type_error; - } - ival = (int)Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - - if (ival > Byte.MaxValue || ival < Byte.MinValue) - { - goto overflow; + result = (byte)num; + return true; } - byte b = (byte)ival; - result = b; - return true; case TypeCode.SByte: -#if PYTHON3 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { - if (Runtime.PyBytes_Size(value) == 1) + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { - op = Runtime.PyBytes_AS_STRING(value); - result = (byte)Marshal.ReadByte(op); - return true; + if (Runtime.PyBytes_Size(value) == 1) + { + op = Runtime.PyBytes_AS_STRING(value); + result = (byte)Marshal.ReadByte(op); + return true; + } + goto type_error; } - goto type_error; - } -#elif PYTHON2 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) - { - if (Runtime.PyString_Size(value) == 1) + + int num = Runtime.PyLong_AsLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) { - op = Runtime.PyString_AsString(value); - result = (sbyte)Marshal.ReadByte(op); - return true; + goto convert_error; } - goto type_error; - } -#endif - - op = Runtime.PyNumber_Int(value); - if (op == IntPtr.Zero) - { - if (Exceptions.ExceptionMatches(overflow)) + if (num > SByte.MaxValue || num < SByte.MinValue) { goto overflow; } - goto type_error; - } - ival = (int)Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - - if (ival > SByte.MaxValue || ival < SByte.MinValue) - { - goto overflow; + result = (sbyte)num; + return true; } - sbyte sb = (sbyte)ival; - result = sb; - return true; case TypeCode.Char: -#if PYTHON3 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { - if (Runtime.PyBytes_Size(value) == 1) + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { - op = Runtime.PyBytes_AS_STRING(value); - result = (byte)Marshal.ReadByte(op); - return true; + if (Runtime.PyBytes_Size(value) == 1) + { + op = Runtime.PyBytes_AS_STRING(value); + result = (byte)Marshal.ReadByte(op); + return true; + } + goto type_error; } - goto type_error; - } -#elif PYTHON2 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) - { - if (Runtime.PyString_Size(value) == 1) + else if (Runtime.PyObject_TypeCheck(value, Runtime.PyUnicodeType)) { - op = Runtime.PyString_AsString(value); - result = (char)Marshal.ReadByte(op); - return true; + if (Runtime.PyUnicode_GetSize(value) == 1) + { + op = Runtime.PyUnicode_AsUnicode(value); + Char[] buff = new Char[1]; + Marshal.Copy(op, buff, 0, 1); + result = buff[0]; + return true; + } + goto type_error; } - goto type_error; - } -#endif - else if (Runtime.PyObject_TypeCheck(value, Runtime.PyUnicodeType)) - { - if (Runtime.PyUnicode_GetSize(value) == 1) + int num = Runtime.PyLong_AsLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) { - op = Runtime.PyUnicode_AsUnicode(value); - Char[] buff = new Char[1]; - Marshal.Copy(op, buff, 0, 1); - result = buff[0]; - return true; + goto convert_error; } - goto type_error; - } - - op = Runtime.PyNumber_Int(value); - if (op == IntPtr.Zero) - { - goto type_error; - } - ival = Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - if (ival > Char.MaxValue || ival < Char.MinValue) - { - goto overflow; + if (num > Char.MaxValue || num < Char.MinValue) + { + goto overflow; + } + result = (char)num; + return true; } - result = (char)ival; - return true; case TypeCode.Int16: - op = Runtime.PyNumber_Int(value); - if (op == IntPtr.Zero) { - if (Exceptions.ExceptionMatches(overflow)) + int num = Runtime.PyLong_AsLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > Int16.MaxValue || num < Int16.MinValue) { goto overflow; } - goto type_error; - } - ival = (int)Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - if (ival > Int16.MaxValue || ival < Int16.MinValue) - { - goto overflow; + result = (short)num; + return true; } - short s = (short)ival; - result = s; - return true; case TypeCode.Int64: - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) { - if (Exceptions.ExceptionMatches(overflow)) + long num = (long)Runtime.PyLong_AsLongLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) { - goto overflow; + goto convert_error; } - goto type_error; - } - long l = (long)Runtime.PyLong_AsLongLong(op); - Runtime.XDecref(op); - if ((l == -1) && Exceptions.ErrorOccurred()) - { - goto overflow; + result = num; + return true; } - result = l; - return true; case TypeCode.UInt16: - op = Runtime.PyNumber_Int(value); - if (op == IntPtr.Zero) { - if (Exceptions.ExceptionMatches(overflow)) + long num = Runtime.PyLong_AsLong(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > UInt16.MaxValue || num < UInt16.MinValue) { goto overflow; } - goto type_error; - } - ival = (int)Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - if (ival > UInt16.MaxValue || ival < UInt16.MinValue) - { - goto overflow; + result = (ushort)num; + return true; } - ushort us = (ushort)ival; - result = us; - return true; case TypeCode.UInt32: - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) { - if (Exceptions.ExceptionMatches(overflow)) + op = value; + if (Runtime.PyObject_TYPE(value) != Runtime.PyLongType) { - goto overflow; + op = Runtime.PyNumber_Long(value); + if (op == IntPtr.Zero) + { + goto convert_error; + } } - goto type_error; - } - uint ui = (uint)Runtime.PyLong_AsUnsignedLong(op); - - if (Exceptions.ErrorOccurred()) - { - Runtime.XDecref(op); - goto overflow; - } - - IntPtr check = Runtime.PyLong_FromUnsignedLong(ui); - int err = Runtime.PyObject_Compare(check, op); - Runtime.XDecref(check); - Runtime.XDecref(op); - if (0 != err || Exceptions.ErrorOccurred()) - { - goto overflow; + if (Runtime.Is32Bit || Runtime.IsWindows) + { + uint num = Runtime.PyLong_AsUnsignedLong32(op); + if (num == uint.MaxValue && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = num; + } + else + { + ulong num = Runtime.PyLong_AsUnsignedLong64(op); + if (num == ulong.MaxValue && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + try + { + result = Convert.ToUInt32(num); + } + catch (OverflowException) + { + // Probably wasn't an overflow in python but was in C# (e.g. if cpython + // longs are 64 bit then 0xFFFFFFFF + 1 will not overflow in + // PyLong_AsUnsignedLong) + goto overflow; + } + } + return true; } - result = ui; - return true; - case TypeCode.UInt64: - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) { - if (Exceptions.ExceptionMatches(overflow)) + op = value; + if (Runtime.PyObject_TYPE(value) != Runtime.PyLongType) + { + op = Runtime.PyNumber_Long(value); + if (op == IntPtr.Zero) + { + goto convert_error; + } + } + ulong num = Runtime.PyLong_AsUnsignedLongLong(op); + if (num == ulong.MaxValue && Exceptions.ErrorOccurred()) { goto overflow; } - goto type_error; - } - ulong ul = (ulong)Runtime.PyLong_AsUnsignedLongLong(op); - Runtime.XDecref(op); - if (Exceptions.ErrorOccurred()) - { - goto overflow; + result = num; + return true; } - result = ul; - return true; - case TypeCode.Single: - op = Runtime.PyNumber_Float(value); - if (op == IntPtr.Zero) { - if (Exceptions.ExceptionMatches(overflow)) + double num = Runtime.PyFloat_AsDouble(value); + if (num == -1.0 && Exceptions.ErrorOccurred()) { - goto overflow; + goto convert_error; } - goto type_error; - } - double dd = Runtime.PyFloat_AsDouble(op); - Runtime.XDecref(op); - if (dd > Single.MaxValue || dd < Single.MinValue) - { - goto overflow; + if (num > Single.MaxValue || num < Single.MinValue) + { + if (!double.IsInfinity(num)) + { + goto overflow; + } + } + result = (float)num; + return true; } - result = (float)dd; - return true; case TypeCode.Double: - op = Runtime.PyNumber_Float(value); - if (op == IntPtr.Zero) { - goto type_error; - } - double d = Runtime.PyFloat_AsDouble(op); - Runtime.XDecref(op); - if (d > Double.MaxValue || d < Double.MinValue) - { - goto overflow; + double num = Runtime.PyFloat_AsDouble(value); + if (num == -1.0 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = num; + return true; } - result = d; - return true; + default: + goto type_error; } + convert_error: + if (op != value) + { + Runtime.XDecref(op); + } + if (!setError) + { + Exceptions.Clear(); + } + return false; - type_error: - + type_error: if (setError) { string tpName = Runtime.PyObject_GetTypeName(value); Exceptions.SetError(Exceptions.TypeError, $"'{tpName}' value cannot be converted to {obType}"); } - return false; - overflow: - + overflow: + // C# level overflow error + if (op != value) + { + Runtime.XDecref(op); + } if (setError) { Exceptions.SetError(Exceptions.OverflowError, "value too large to convert"); } - return false; } @@ -799,17 +802,20 @@ private static void SetConversionError(IntPtr value, Type target) /// /// Convert a Python value to a correctly typed managed array instance. - /// The Python value must support the Python sequence protocol and the + /// The Python value must support the Python iterator protocol or and the /// items in the sequence must be convertible to the target array type. /// private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - int size = Runtime.PySequence_Size(value); result = null; - if (size < 0) - { + bool IsSeqObj = Runtime.PySequence_Check(value); + var len = IsSeqObj ? Runtime.PySequence_Size(value) : -1; + + IntPtr IterObject = Runtime.PyObject_GetIter(value); + + if(IterObject==IntPtr.Zero) { if (setError) { SetConversionError(value, obType); @@ -817,21 +823,17 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - Array items = Array.CreateInstance(elementType, size); + Array items; - // XXX - is there a better way to unwrap this if it is a real array? - for (var i = 0; i < size; i++) + var listType = typeof(List<>); + var constructedListType = listType.MakeGenericType(elementType); + IList list = IsSeqObj ? (IList) Activator.CreateInstance(constructedListType, new Object[] {(int) len}) : + (IList) Activator.CreateInstance(constructedListType); + IntPtr item; + + while ((item = Runtime.PyIter_Next(IterObject)) != IntPtr.Zero) { object obj = null; - IntPtr item = Runtime.PySequence_GetItem(value, i); - if (item == IntPtr.Zero) - { - if (setError) - { - SetConversionError(value, obType); - return false; - } - } if (!Converter.ToManaged(item, elementType, out obj, true)) { @@ -839,9 +841,13 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - items.SetValue(obj, i); + list.Add(obj); Runtime.XDecref(item); } + Runtime.XDecref(IterObject); + + items = Array.CreateInstance(elementType, list.Count); + list.CopyTo(items, 0); result = items; return true; diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs new file mode 100644 index 000000000..b10d0c59f --- /dev/null +++ b/src/runtime/converterextensions.cs @@ -0,0 +1,189 @@ +namespace Python.Runtime +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using Python.Runtime.Codecs; + + /// + /// Defines conversion to CLR types (unmarshalling) + /// + public interface IPyObjectDecoder + { + /// + /// Checks if this decoder can decode from to + /// + bool CanDecode(PyObject objectType, Type targetType); + /// + /// Attempts do decode into a variable of specified type + /// + /// CLR type to decode into + /// Object to decode + /// The variable, that will receive decoding result + /// + bool TryDecode(PyObject pyObj, out T value); + } + + /// + /// Defines conversion from CLR objects into Python objects (e.g. ) (marshalling) + /// + public interface IPyObjectEncoder + { + /// + /// Checks if encoder can encode CLR objects of specified type + /// + bool CanEncode(Type type); + /// + /// Attempts to encode CLR object into Python object + /// + PyObject TryEncode(object value); + } + + /// + /// This class allows to register additional marshalling codecs. + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static class PyObjectConversions + { + static readonly DecoderGroup decoders = new DecoderGroup(); + static readonly EncoderGroup encoders = new EncoderGroup(); + + /// + /// Registers specified encoder (marshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterEncoder(IPyObjectEncoder encoder) + { + if (encoder == null) throw new ArgumentNullException(nameof(encoder)); + + lock (encoders) + { + encoders.Add(encoder); + } + } + + /// + /// Registers specified decoder (unmarshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterDecoder(IPyObjectDecoder decoder) + { + if (decoder == null) throw new ArgumentNullException(nameof(decoder)); + + lock (decoders) + { + decoders.Add(decoder); + } + } + + #region Encoding + internal static PyObject TryEncode(object obj, Type type) + { + if (obj == null) throw new ArgumentNullException(nameof(obj)); + if (type == null) throw new ArgumentNullException(nameof(type)); + + foreach (var encoder in clrToPython.GetOrAdd(type, GetEncoders)) + { + var result = encoder.TryEncode(obj); + if (result != null) return result; + } + + return null; + } + + static readonly ConcurrentDictionary + clrToPython = new ConcurrentDictionary(); + static IPyObjectEncoder[] GetEncoders(Type type) + { + lock (encoders) + { + return encoders.GetEncoders(type).ToArray(); + } + } + #endregion + + #region Decoding + static readonly ConcurrentDictionary + pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) + { + if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); + if (pyType == IntPtr.Zero) throw new ArgumentNullException(nameof(pyType)); + if (targetType == null) throw new ArgumentNullException(nameof(targetType)); + + var decoder = pythonToClr.GetOrAdd(new TypePair(pyType, targetType), pair => GetDecoder(pair.PyType, pair.ClrType)); + result = null; + if (decoder == null) return false; + return decoder.Invoke(pyHandle, out result); + } + + static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type targetType) + { + IPyObjectDecoder decoder; + using (var pyType = new PyObject(Runtime.SelfIncRef(sourceType))) + { + lock (decoders) + { + decoder = decoders.GetDecoder(pyType, targetType); + if (decoder == null) return null; + } + } + + var decode = genericDecode.MakeGenericMethod(targetType); + + bool TryDecode(IntPtr pyHandle, out object result) + { + var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle)); + var @params = new object[] { pyObj, null }; + bool success = (bool)decode.Invoke(decoder, @params); + if (!success) + { + pyObj.Dispose(); + } + + result = @params[1]; + return success; + } + + return TryDecode; + } + + static readonly MethodInfo genericDecode = typeof(IPyObjectDecoder).GetMethod(nameof(IPyObjectDecoder.TryDecode)); + + #endregion + + internal static void Reset() + { + lock (encoders) + lock (decoders) + { + clrToPython.Clear(); + pythonToClr.Clear(); + encoders.Clear(); + decoders.Clear(); + } + } + + struct TypePair : IEquatable + { + internal readonly IntPtr PyType; + internal readonly Type ClrType; + + public TypePair(IntPtr pyType, Type clrType) + { + this.PyType = pyType; + this.ClrType = clrType; + } + + public override int GetHashCode() + => this.ClrType.GetHashCode() ^ this.PyType.GetHashCode(); + + public bool Equals(TypePair other) + => this.PyType == other.PyType && this.ClrType == other.ClrType; + + public override bool Equals(object obj) => obj is TypePair other && this.Equals(other); + } + } +} diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs index 2a91a74b4..5e854bffd 100644 --- a/src/runtime/debughelper.cs +++ b/src/runtime/debughelper.cs @@ -60,13 +60,13 @@ internal static void DumpType(IntPtr type) //DebugUtil.Print(" mro: ", op); - FieldInfo[] slots = typeof(TypeOffset).GetFields(); + var slots = TypeOffset.GetOffsets(); int size = IntPtr.Size; - for (var i = 0; i < slots.Length; i++) + foreach (var entry in slots) { - int offset = i * size; - name = slots[i].Name; + int offset = entry.Value; + name = entry.Key; op = Marshal.ReadIntPtr(type, offset); Console.WriteLine(" {0}: {1}", name, op); } @@ -116,10 +116,10 @@ internal static void debug(string msg) Console.WriteLine(" {0}", msg); } - /// + /// /// Helper function to inspect/compare managed to native conversions. - /// Especially useful when debugging CustomMarshaler. - /// + /// Especially useful when debugging CustomMarshaler. + /// /// [Conditional("DEBUG")] public static void PrintHexBytes(byte[] bytes) @@ -137,5 +137,12 @@ public static void PrintHexBytes(byte[] bytes) Console.WriteLine(); } } + + [Conditional("DEBUG")] + public static void AssertHasReferences(IntPtr obj) + { + long refcount = Runtime.Refcount(obj); + Debug.Assert(refcount > 0, "Object refcount is 0 or less"); + } } } diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index df5eec427..3e6541c44 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -185,6 +185,8 @@ public class Dispatcher { public IntPtr target; public Type dtype; + private bool _disposed = false; + private bool _finalized = false; public Dispatcher(IntPtr target, Type dtype) { @@ -195,32 +197,41 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) + if (_finalized || _disposed) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(ref target); + } + + public void Dispose() + { + if (_disposed) + { + return; } + _disposed = true; + Runtime.XDecref(target); + target = IntPtr.Zero; + dtype = null; + GC.SuppressFinalize(this); } public object Dispatch(ArrayList args) { IntPtr gs = PythonEngine.AcquireLock(); - object ob = null; + object ob; try { ob = TrueDispatch(args); } - catch (Exception e) + finally { PythonEngine.ReleaseLock(gs); - throw e; } - PythonEngine.ReleaseLock(gs); return ob; } @@ -253,27 +264,15 @@ public object TrueDispatch(ArrayList args) return null; } - object result = null; - if (!Converter.ToManaged(op, rtype, out result, false)) + object result; + if (!Converter.ToManaged(op, rtype, out result, true)) { Runtime.XDecref(op); - throw new ConversionException($"could not convert Python result to {rtype}"); + throw new PythonException(); } Runtime.XDecref(op); return result; } } - - - public class ConversionException : Exception - { - public ConversionException() - { - } - - public ConversionException(string msg) : base(msg) - { - } - } } diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index e1103cbc7..e0d29f1a0 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// Each of those type objects is associated an instance of this class, /// which provides its implementation. /// + [Serializable] internal class DelegateObject : ClassBase { private MethodBinder binder; @@ -51,6 +52,12 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (DelegateObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type type = self.type.Value; + if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("class takes exactly one argument"); @@ -63,7 +70,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("argument must be callable"); } - Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type, method); + Delegate d = PythonEngine.DelegateManager.GetDelegate(type, method); return CLRObject.GetInstHandle(d, self.pyHandle); } @@ -96,7 +103,6 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) /// /// Implements __cmp__ for reflected delegate types. /// -#if PYTHON3 // TODO: Doesn't PY2 implement tp_richcompare too? public new static IntPtr tp_richcompare(IntPtr ob, IntPtr other, int op) { if (op != Runtime.Py_EQ && op != Runtime.Py_NE) @@ -126,13 +132,5 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) Runtime.XIncref(pyfalse); return pyfalse; } -#elif PYTHON2 - public static int tp_compare(IntPtr ob, IntPtr other) - { - Delegate d1 = GetTrueDelegate(ob); - Delegate d2 = GetTrueDelegate(other); - return d1 == d2 ? 0 : -1; - } -#endif } } diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index b8b4c82ad..581095185 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -5,6 +5,7 @@ namespace Python.Runtime /// /// Implements a Python event binding type, similar to a method binding. /// + [Serializable] internal class EventBinding : ExtensionType { private EventObject e; @@ -118,7 +119,14 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (EventBinding)GetManagedObject(ob); Runtime.XDecref(self.target); - ExtensionType.FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (EventBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.target); + return 0; } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 5f18c4609..0f2796a14 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR events. /// + [Serializable] internal class EventObject : ExtensionType { internal string name; @@ -202,7 +203,7 @@ public static IntPtr tp_repr(IntPtr ob) { Runtime.XDecref(self.unbound.pyHandle); } - FinalizeObject(self); + self.Dealloc(); } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 9023cfcfa..ab28905d2 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -15,6 +15,7 @@ namespace Python.Runtime /// it subclasses System.Object. Instead TypeManager.CreateType() uses /// Python's exception.Exception class as base class for System.Exception. /// + [Serializable] internal class ExceptionClassObject : ClassObject { internal ExceptionClassObject(Type tp) : base(tp) @@ -36,6 +37,29 @@ internal static Exception ToException(IntPtr ob) return e; } + /// + /// Exception __repr__ implementation + /// + public new static IntPtr tp_repr(IntPtr ob) + { + Exception e = ToException(ob); + if (e == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + string name = e.GetType().Name; + string message; + if (e.Message != String.Empty) + { + message = String.Format("{0}('{1}')", name, e.Message); + } + else + { + message = String.Format("{0}()", name); + } + return Runtime.PyUnicode_FromString(message); + } + /// /// Exception __str__ implementation /// @@ -66,21 +90,17 @@ internal static Exception ToException(IntPtr ob) /// /// Readability of the Exceptions class improvements as we look toward version 2.7 ... /// - public class Exceptions + public static class Exceptions { internal static IntPtr warnings_module; internal static IntPtr exceptions_module; - private Exceptions() - { - } - /// /// Initialization performed on startup of the Python runtime. /// internal static void Initialize() { - string exceptionsModuleName = Runtime.IsPython3 ? "builtins" : "exceptions"; + string exceptionsModuleName = "builtins"; exceptions_module = Runtime.PyImport_ImportModule(exceptionsModuleName); Exceptions.ErrorCheck(exceptions_module); @@ -109,21 +129,23 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) + { + return; + } + Type type = typeof(Exceptions); + foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) { - Type type = typeof(Exceptions); - foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) + var op = (IntPtr)fi.GetValue(type); + if (op == IntPtr.Zero) { - var op = (IntPtr)fi.GetValue(type); - if (op != IntPtr.Zero) - { - Runtime.XDecref(op); - } + continue; } - Runtime.XDecref(exceptions_module); - Runtime.PyObject_HasAttrString(warnings_module, "xx"); - Runtime.XDecref(warnings_module); + Runtime.XDecref(op); + fi.SetValue(null, IntPtr.Zero); } + Runtime.Py_CLEAR(ref exceptions_module); + Runtime.Py_CLEAR(ref warnings_module); } /// @@ -157,13 +179,11 @@ internal static void SetArgsAndCause(IntPtr ob) Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); -#if PYTHON3 if (e.InnerException != null) { IntPtr cause = CLRObject.GetInstHandle(e.InnerException); Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); } -#endif } /// @@ -233,9 +253,9 @@ public static void SetError(IntPtr ob, string value) /// Sets the current Python exception given a Python object. /// This is a wrapper for the Python PyErr_SetObject call. /// - public static void SetError(IntPtr ob, IntPtr value) + public static void SetError(IntPtr type, IntPtr exceptionObject) { - Runtime.PyErr_SetObject(ob, value); + Runtime.PyErr_SetObject(new BorrowedReference(type), new BorrowedReference(exceptionObject)); } /// @@ -256,13 +276,16 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.PyErr_SetObject(pe.PyType, pe.PyValue); + Runtime.XIncref(pe.PyType); + Runtime.XIncref(pe.PyValue); + Runtime.XIncref(pe.PyTB); + Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); return; } IntPtr op = CLRObject.GetInstHandle(e); - IntPtr etype = Runtime.PyObject_GetAttrString(op, "__class__"); - Runtime.PyErr_SetObject(etype, op); + IntPtr etype = Runtime.PyObject_GetAttr(op, PyIdentifier.__class__); + Runtime.PyErr_SetObject(new BorrowedReference(etype), new BorrowedReference(op)); Runtime.XDecref(etype); Runtime.XDecref(op); } @@ -276,7 +299,7 @@ public static void SetError(Exception e) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != 0; + return Runtime.PyErr_Occurred() != IntPtr.Zero; } /// @@ -360,9 +383,6 @@ puplic static variables on the Exceptions class filled in from public static IntPtr Exception; public static IntPtr StopIteration; public static IntPtr GeneratorExit; -#if PYTHON2 - public static IntPtr StandardError; -#endif public static IntPtr ArithmeticError; public static IntPtr LookupError; diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 9569b0485..a5f0f1219 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// type object, such as the types that represent CLR methods, fields, /// etc. Instances implemented by this class do not support sub-typing. /// + [Serializable] internal abstract class ExtensionType : ManagedType { public ExtensionType() @@ -28,19 +29,24 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - GCHandle gc = GCHandle.Alloc(this); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); + // Steals a ref to tpHandle. + tpHandle = tp; + pyHandle = py; + + SetupGc(); + } + + void SetupGc () + { + GCHandle gc = AllocGCHandle(TrackTypes.Extension); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most // concrete extension types, so untrack the object to save calls // from Python into the managed runtime that are pure overhead. - Runtime.PyObject_GC_UnTrack(py); - - tpHandle = tp; - pyHandle = py; - gcHandle = gc; + Runtime.PyObject_GC_UnTrack(pyHandle); } @@ -49,11 +55,16 @@ public ExtensionType() /// public static void FinalizeObject(ManagedType self) { + ClearObjectDict(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); - self.gcHandle.Free(); + // Not necessary for decref of `tpHandle`. + self.FreeGCHandle(); } + protected void Dealloc() + { + FinalizeObject(this); + } /// /// Type __setattr__ implementation. @@ -81,27 +92,6 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } - /// - /// Required Python GC support. - /// - public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) - { - return 0; - } - - - public static int tp_clear(IntPtr ob) - { - return 0; - } - - - public static int tp_is_gc(IntPtr type) - { - return 1; - } - - /// /// Default dealloc implementation. /// @@ -109,8 +99,14 @@ public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. - ManagedType self = GetManagedObject(ob); - FinalizeObject(self); + var self = (ExtensionType)GetManagedObject(ob); + self.Dealloc(); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + SetupGc(); } } } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 7c9a466d5..2850ac6e1 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -3,12 +3,14 @@ namespace Python.Runtime { + using MaybeFieldInfo = MaybeMemberInfo; /// /// Implements a Python descriptor type that provides access to CLR fields. /// + [Serializable] internal class FieldObject : ExtensionType { - private FieldInfo info; + private MaybeFieldInfo info; public FieldObject(FieldInfo info) { @@ -29,8 +31,13 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return IntPtr.Zero; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return IntPtr.Zero; + } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (ob == IntPtr.Zero || ob == Runtime.PyNone) { @@ -55,6 +62,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return IntPtr.Zero; + } result = info.GetValue(co.inst); return Converter.ToPython(result, info.FieldType); } @@ -79,6 +91,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return -1; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return -1; + } if (val == IntPtr.Zero) { @@ -86,7 +103,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) return -1; } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (info.IsLiteral || info.IsInitOnly) { @@ -115,6 +132,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) if (!is_static) { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return -1; + } info.SetValue(co.inst, newval); } else @@ -136,7 +158,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (FieldObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs new file mode 100644 index 000000000..fe2e46aac --- /dev/null +++ b/src/runtime/finalizer.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public class ErrorArgs : EventArgs + { + public Exception Error { get; set; } + } + + public static readonly Finalizer Instance = new Finalizer(); + + public event EventHandler CollectOnce; + public event EventHandler ErrorHandler; + + public int Threshold { get; set; } + public bool Enable { get; set; } + + private ConcurrentQueue _objQueue = new ConcurrentQueue(); + private int _throttled; + + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + public bool RefCountValidationEnabled { get; set; } = true; +#else + public readonly bool RefCountValidationEnabled = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + public class IncorrectFinalizeArgs : EventArgs + { + public IntPtr Handle { get; internal set; } + public ICollection ImpactedObjects { get; internal set; } + } + + public class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + private string _message; + public override string Message => _message; + + public IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + string name = Runtime.GetManagedString(pyname); + Runtime.XDecref(pyname); + _message = $"<{name}> may has a incorrect ref count"; + } + } + + public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + #pragma warning disable 414 + public event IncorrectRefCntHandler IncorrectRefCntResolver = null; + #pragma warning restore 414 + public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + + private Finalizer() + { + Enable = true; + Threshold = 200; + } + + public void Collect() => this.DisposeAll(); + + internal void ThrottledCollect() + { + _throttled = unchecked(this._throttled + 1); + if (!Enable || _throttled < Threshold) return; + _throttled = 0; + this.Collect(); + } + + internal List GetCollectedObjects() + { + return _objQueue.ToList(); + } + + internal void AddFinalizedObject(ref IntPtr obj) + { + if (!Enable || obj == IntPtr.Zero) + { + return; + } + +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + this._objQueue.Enqueue(obj); + } + obj = IntPtr.Zero; + } + + internal static void Shutdown() + { + Instance.DisposeAll(); + } + + private void DisposeAll() + { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + IntPtr obj; + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + + try + { + while (!_objQueue.IsEmpty) + { + if (!_objQueue.TryDequeue(out obj)) + continue; + + Runtime.XDecref(obj); + try + { + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + var handler = ErrorHandler; + if (handler is null) + { + throw new FinalizationException( + "Python object finalization failed", + disposable: obj, innerException: e); + } + + handler.Invoke(this, new ErrorArgs() + { + Error = e + }); + } + } + } + finally + { + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType, errVal, traceback); + } + } + } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + var handle = obj; + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResolver != null) + { + var funcList = IncorrectRefCntResolver.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif + } + + public class FinalizationException : Exception + { + public IntPtr PythonObject { get; } + + public FinalizationException(string message, IntPtr disposable, Exception innerException) + : base(message, innerException) + { + if (disposable == IntPtr.Zero) throw new ArgumentNullException(nameof(disposable)); + this.PythonObject = disposable; + } + } +} diff --git a/src/runtime/generictype.cs b/src/runtime/generictype.cs index eeae801d2..76d2e9a5d 100644 --- a/src/runtime/generictype.cs +++ b/src/runtime/generictype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// generic types. Both are essentially factories for creating closed /// types based on the required generic type parameters. /// + [Serializable] internal class GenericType : ClassBase { internal GenericType(Type tp) : base(tp) diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 9772d082f..92b847e98 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -1,5 +1,7 @@ +using System.Linq; using System; using System.Collections.Generic; +using System.Resources; namespace Python.Runtime { @@ -7,15 +9,14 @@ namespace Python.Runtime /// This class is responsible for efficiently maintaining the bits /// of information we need to support aliases with 'nice names'. /// - internal class GenericUtil + internal static class GenericUtil { + /// + /// Maps namespace -> generic base name -> list of generic type names + /// private static Dictionary>> mapping; - private GenericUtil() - { - } - - static GenericUtil() + public static void Reset() { mapping = new Dictionary>>(); } @@ -23,6 +24,7 @@ static GenericUtil() /// /// Register a generic type that appears in a given namespace. /// + /// A generic type definition (t.IsGenericTypeDefinition must be true) internal static void Register(Type t) { if (null == t.Namespace || null == t.Name) @@ -30,22 +32,15 @@ internal static void Register(Type t) return; } - Dictionary> nsmap = null; - mapping.TryGetValue(t.Namespace, out nsmap); - if (nsmap == null) + Dictionary> nsmap; + if (!mapping.TryGetValue(t.Namespace, out nsmap)) { nsmap = new Dictionary>(); mapping[t.Namespace] = nsmap; } - string basename = t.Name; - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } - List gnames = null; - nsmap.TryGetValue(basename, out gnames); - if (gnames == null) + string basename = GetBasename(t.Name); + List gnames; + if (!nsmap.TryGetValue(basename, out gnames)) { gnames = new List(); nsmap[basename] = gnames; @@ -58,9 +53,8 @@ internal static void Register(Type t) /// public static List GetGenericBaseNames(string ns) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) { return null; } @@ -73,64 +67,41 @@ public static List GetGenericBaseNames(string ns) } /// - /// xxx + /// Finds a generic type with the given number of generic parameters and the same name and namespace as . /// public static Type GenericForType(Type t, int paramCount) { return GenericByName(t.Namespace, t.Name, paramCount); } - public static Type GenericByName(string ns, string name, int paramCount) - { - foreach (Type t in GenericsByName(ns, name)) - { - if (t.GetGenericArguments().Length == paramCount) - { - return t; - } - } - return null; - } - - public static List GenericsForType(Type t) - { - return GenericsByName(t.Namespace, t.Name); - } - - public static List GenericsByName(string ns, string basename) + /// + /// Finds a generic type in the given namespace with the given name and number of generic parameters. + /// + public static Type GenericByName(string ns, string basename, int paramCount) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) { return null; } - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } - - List names = null; - nsmap.TryGetValue(basename, out names); - if (names == null) + List names; + if (!nsmap.TryGetValue(GetBasename(basename), out names)) { return null; } - var result = new List(); foreach (string name in names) { string qname = ns + "." + name; - Type o = AssemblyManager.LookupType(qname); - if (o != null) + Type o = AssemblyManager.LookupTypes(qname).FirstOrDefault(); + if (o != null && o.GetGenericArguments().Length == paramCount) { - result.Add(o); + return o; } } - return result; + return null; } /// @@ -138,13 +109,12 @@ public static List GenericsByName(string ns, string basename) /// public static string GenericNameForBaseName(string ns, string name) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) { return null; } - List gnames = null; + List gnames; nsmap.TryGetValue(name, out gnames); if (gnames?.Count > 0) { @@ -152,5 +122,18 @@ public static string GenericNameForBaseName(string ns, string name) } return null; } + + private static string GetBasename(string name) + { + int tick = name.IndexOf("`"); + if (tick > -1) + { + return name.Substring(0, tick); + } + else + { + return name; + } + } } } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index bc9ac5eee..af6174188 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Runtime.InteropServices; namespace Python.Runtime @@ -6,14 +7,13 @@ namespace Python.Runtime /// /// Implements the "import hook" used to integrate Python with the CLR. /// - internal class ImportHook + internal static class ImportHook { private static IntPtr py_import; private static CLRModule root; private static MethodWrapper hook; private static IntPtr py_clr_module; -#if PYTHON3 private static IntPtr module_def = IntPtr.Zero; internal static void InitializeModuleDef() @@ -23,31 +23,64 @@ internal static void InitializeModuleDef() module_def = ModuleDefOffset.AllocModuleDef("clr"); } } -#endif + + internal static void ReleaseModuleDef() + { + if (module_def == IntPtr.Zero) + { + return; + } + ModuleDefOffset.FreeModuleDef(module_def); + module_def = IntPtr.Zero; + } /// - /// Initialization performed on startup of the Python runtime. + /// Initialize just the __import__ hook itself. /// - internal static void Initialize() + static void InitImport() { - // Initialize the Python <--> CLR module hook. We replace the - // built-in Python __import__ with our own. This isn't ideal, - // but it provides the most "Pythonic" way of dealing with CLR - // modules (Python doesn't provide a way to emulate packages). - IntPtr dict = Runtime.PyImport_GetModuleDict(); + // We replace the built-in Python __import__ with our own: first + // look in CLR modules, then if we don't find any call the default + // Python __import__. + IntPtr builtins = Runtime.GetBuiltins(); + py_import = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); + PythonException.ThrowIfIsNull(py_import); - IntPtr mod = Runtime.IsPython3 - ? Runtime.PyImport_ImportModule("builtins") - : Runtime.PyDict_GetItemString(dict, "__builtin__"); - - py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - Runtime.PyObject_SetAttrString(mod, "__import__", hook.ptr); - Runtime.XDecref(hook.ptr); + int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, hook.ptr); + PythonException.ThrowIfIsNotZero(res); + + Runtime.XDecref(builtins); + } + + /// + /// Restore the __import__ hook. + /// + static void RestoreImport() + { + IntPtr builtins = Runtime.GetBuiltins(); + + int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); + PythonException.ThrowIfIsNotZero(res); + Runtime.XDecref(py_import); + py_import = IntPtr.Zero; + + hook.Release(); + hook = null; + + Runtime.XDecref(builtins); + } + + /// + /// Initialization performed on startup of the Python runtime. + /// + internal static void Initialize() + { + InitImport(); + // Initialize the clr module and tell Python about it. root = new CLRModule(); -#if PYTHON3 // create a python module with the same methods as the clr module-like object InitializeModuleDef(); py_clr_module = Runtime.PyModule_Create2(module_def, 3); @@ -58,10 +91,7 @@ internal static void Initialize() clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); Runtime.PyDict_Update(mod_dict, clr_dict); -#elif PYTHON2 - Runtime.XIncref(root.pyHandle); // we are using the module two times - py_clr_module = root.pyHandle; // Alias handle for PY2/PY3 -#endif + IntPtr dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); } @@ -72,12 +102,42 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) { - Runtime.XDecref(py_clr_module); - Runtime.XDecref(root.pyHandle); - Runtime.XDecref(py_import); + return; } + + RestoreImport(); + + bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; + if (shouldFreeDef) + { + ReleaseModuleDef(); + } + + Runtime.XDecref(root.pyHandle); + root = null; + CLRModule.Reset(); + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + // Increment the reference counts here so that the objects don't + // get freed in Shutdown. + Runtime.XIncref(py_clr_module); + Runtime.XIncref(root.pyHandle); + storage.AddValue("py_clr_module", py_clr_module); + storage.AddValue("root", root.pyHandle); + } + + internal static void RestoreRuntimeData(RuntimeDataStorage storage) + { + InitImport(); + storage.GetValue("py_clr_module", out py_clr_module); + var rootHandle = storage.GetValue("root"); + root = (CLRModule)ManagedType.GetManagedObject(rootHandle); } /// @@ -87,13 +147,6 @@ public static IntPtr GetCLRModule(IntPtr? fromList = null) { root.InitializePreload(); - if (Runtime.IsPython2) - { - Runtime.XIncref(py_clr_module); - return py_clr_module; - } - - // Python 3 // update the module dictionary with the contents of the root dictionary root.LoadNames(); IntPtr py_mod_dict = Runtime.PyModule_GetDict(py_clr_module); @@ -155,7 +208,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // hook is saved as this.py_import. This version handles CLR // import and defers to the normal builtin for everything else. - int num_args = Runtime.PyTuple_Size(args); + var num_args = Runtime.PyTuple_Size(args); if (num_args < 1) { return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); @@ -201,86 +254,50 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } return clr_module; } - if (mod_name == "CLR") + + string realname = mod_name; + string clr_prefix = null; + + // 2010-08-15: Always seemed smart to let python try first... + // This shaves off a few tenths of a second on test_module.py + // and works around a quirk where 'sys' is found by the + // LoadImplicit() deprecation logic. + // Turns out that the AssemblyManager.ResolveHandler() checks to see if any + // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very + // little sense to me. + IntPtr res = Runtime.PyObject_Call(py_import, args, kw); + if (res != IntPtr.Zero) { - Exceptions.deprecation("The CLR module is deprecated. Please use 'clr'."); - IntPtr clr_module = GetCLRModule(fromList); - if (clr_module != IntPtr.Zero) + // There was no error. + if (fromlist && IsLoadAll(fromList)) { - IntPtr sys_modules = Runtime.PyImport_GetModuleDict(); - if (sys_modules != IntPtr.Zero) - { - Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); - } + var mod = ManagedType.GetManagedObject(res) as ModuleObject; + mod?.LoadNames(); } - return clr_module; + return res; } - string realname = mod_name; - string clr_prefix = null; - if (mod_name.StartsWith("CLR.")) + // There was an error + if (!Exceptions.ExceptionMatches(Exceptions.ImportError)) { - clr_prefix = "CLR."; // prepend when adding the module to sys.modules - realname = mod_name.Substring(4); - string msg = $"Importing from the CLR.* namespace is deprecated. Please import '{realname}' directly."; - Exceptions.deprecation(msg); + // and it was NOT an ImportError; bail out here. + return IntPtr.Zero; } - else - { - // 2010-08-15: Always seemed smart to let python try first... - // This shaves off a few tenths of a second on test_module.py - // and works around a quirk where 'sys' is found by the - // LoadImplicit() deprecation logic. - // Turns out that the AssemblyManager.ResolveHandler() checks to see if any - // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very - // little sense to me. - IntPtr res = Runtime.PyObject_Call(py_import, args, kw); - if (res != IntPtr.Zero) - { - // There was no error. - return res; - } - // There was an error - if (!Exceptions.ExceptionMatches(Exceptions.ImportError)) - { - // and it was NOT an ImportError; bail out here. - return IntPtr.Zero; - } - if (mod_name == string.Empty) - { - // Most likely a missing relative import. - // For example site-packages\bs4\builder\__init__.py uses it to check if a package exists: - // from . import _html5lib - // We don't support them anyway - return IntPtr.Zero; - } - // Otherwise, just clear the it. - Exceptions.Clear(); + if (mod_name == string.Empty) + { + // Most likely a missing relative import. + // For example site-packages\bs4\builder\__init__.py uses it to check if a package exists: + // from . import _html5lib + // We don't support them anyway + return IntPtr.Zero; } + // Save the exception + var originalException = new PythonException(); + // Otherwise, just clear the it. + Exceptions.Clear(); string[] names = realname.Split('.'); - // Now we need to decide if the name refers to a CLR module, - // and may have to do an implicit load (for b/w compatibility) - // using the AssemblyManager. The assembly manager tries - // really hard not to use Python objects or APIs, because - // parts of it can run recursively and on strange threads. - // - // It does need an opportunity from time to time to check to - // see if sys.path has changed, in a context that is safe. Here - // we know we have the GIL, so we'll let it update if needed. - - AssemblyManager.UpdatePath(); - if (!AssemblyManager.IsValidNamespace(realname)) - { - if (!AssemblyManager.LoadImplicit(realname)) - { - // May be called when a module being imported imports a module. - // In particular, I've seen decimal import copy import org.python.core - return Runtime.PyObject_Call(py_import, args, kw); - } - } - // See if sys.modules for this interpreter already has the // requested module. If so, just return the existing module. IntPtr modules = Runtime.PyImport_GetModuleDict(); @@ -290,6 +307,11 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { if (fromlist) { + if (IsLoadAll(fromList)) + { + var mod = ManagedType.GetManagedObject(module) as ModuleObject; + mod?.LoadNames(); + } Runtime.XIncref(module); return module; } @@ -322,7 +344,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) ManagedType mt = tail.GetAttribute(name, true); if (!(mt is ModuleObject)) { - Exceptions.SetError(Exceptions.ImportError, $"No module named {name}"); + originalException.Restore(); return IntPtr.Zero; } if (head == null) @@ -338,27 +360,40 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // Add the module to sys.modules Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.pyHandle); - // If imported from CLR add CLR. to sys.modules as well + // If imported from CLR add clr. to sys.modules as well if (clr_prefix != null) { Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.pyHandle); } } - ModuleObject mod = fromlist ? tail : head; - - if (fromlist && Runtime.PySequence_Size(fromList) == 1) { - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - if (!CLRModule.preload && Runtime.GetManagedString(fp) == "*") + var mod = fromlist ? tail : head; + + if (fromlist && IsLoadAll(fromList)) { mod.LoadNames(); } - Runtime.XDecref(fp); + + Runtime.XIncref(mod.pyHandle); + return mod.pyHandle; } + } - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; + private static bool IsLoadAll(IntPtr fromList) + { + if (CLRModule.preload) + { + return false; + } + if (Runtime.PySequence_Size(fromList) != 1) + { + return false; + } + IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); + bool res = Runtime.GetManagedString(fp) == "*"; + Runtime.XDecref(fp); + return res; } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 7b6d90ca8..0772b57c6 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Bundles the information required to support an indexer property. /// + [Serializable] internal class Indexer { public MethodBinder GetterBinder; @@ -56,7 +57,7 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - int pynargs = Runtime.PyTuple_Size(args); + var pynargs = Runtime.PyTuple_Size(args); MethodBase[] methods = SetterBinder.GetMethods(); if (methods.Length == 0) { @@ -72,7 +73,7 @@ internal bool NeedsDefaultArgs(IntPtr args) return false; } - for (int v = pynargs; v < clrnargs; v++) + for (var v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -95,7 +96,7 @@ internal IntPtr GetDefaultArgs(IntPtr args) { return Runtime.PyTuple_New(0); } - int pynargs = Runtime.PyTuple_Size(args); + var pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple MethodBase[] methods = SetterBinder.GetMethods(); diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index ce1bc9eb0..976c09be0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// Each of those type objects is associated with an instance of this /// class, which provides the implementation for the Python type. /// + [Serializable] internal class InterfaceObject : ClassBase { internal ConstructorInfo ctor; @@ -36,8 +37,12 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); - int nargs = Runtime.PyTuple_Size(args); - Type type = self.type; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + var nargs = Runtime.PyTuple_Size(args); + Type type = self.type.Value; object obj; if (nargs == 1) @@ -71,7 +76,43 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - return CLRObject.GetInstHandle(obj, self.pyHandle); + return self.WrapObject(obj); + } + + /// + /// Wrap the given object in an interface object, so that only methods + /// of the interface are available. + /// + public IntPtr WrapObject(object impl) + { + var objPtr = CLRObject.GetInstHandle(impl, pyHandle); + return objPtr; + } + + /// + /// Expose the wrapped implementation through attributes in both + /// converted/encoded (__implementation__) and raw (__raw_implementation__) form. + /// + public static IntPtr tp_getattro(IntPtr ob, IntPtr key) + { + var clrObj = (CLRObject)GetManagedObject(ob); + + if (!Runtime.PyString_Check(key)) + { + return Exceptions.RaiseTypeError("string expected"); + } + + string name = Runtime.GetManagedString(key); + if (name == "__implementation__") + { + return Converter.ToPython(clrObj.inst); + } + else if (name == "__raw_implementation__") + { + return CLRObject.GetInstHandle(clrObj.inst); + } + + return Runtime.PyObject_GenericGetAttr(ob, key); } } } diff --git a/src/runtime/intern.cs b/src/runtime/intern.cs new file mode 100644 index 000000000..d8bdf863e --- /dev/null +++ b/src/runtime/intern.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Python.Runtime +{ + static partial class InternString + { + private static Dictionary _string2interns; + private static Dictionary _intern2strings; + + static InternString() + { + var identifierNames = typeof(PyIdentifier).GetFields().Select(fi => fi.Name); + var validNames = new HashSet(identifierNames); + if (validNames.Count != _builtinNames.Length) + { + throw new InvalidOperationException("Identifiers args not matching"); + } + foreach (var name in _builtinNames) + { + if (!validNames.Contains(name)) + { + throw new InvalidOperationException($"{name} is not declared"); + } + } + } + + public static void Initialize() + { + _string2interns = new Dictionary(); + _intern2strings = new Dictionary(); + + Type type = typeof(PyIdentifier); + foreach (string name in _builtinNames) + { + IntPtr op = Runtime.PyUnicode_InternFromString(name); + SetIntern(name, op); + type.GetField(name).SetValue(null, op); + } + } + + public static void Shutdown() + { + foreach (var ptr in _intern2strings.Keys) + { + Runtime.XDecref(ptr); + } + _string2interns = null; + _intern2strings = null; + } + + public static string GetManagedString(IntPtr op) + { + string s; + if (TryGetInterned(op, out s)) + { + return s; + } + return Runtime.GetManagedString(op); + } + + public static bool TryGetInterned(IntPtr op, out string s) + { + return _intern2strings.TryGetValue(op, out s); + } + + private static void SetIntern(string s, IntPtr op) + { + _string2interns.Add(s, op); + _intern2strings.Add(op, s); + } + } +} diff --git a/src/runtime/intern_.cs b/src/runtime/intern_.cs new file mode 100644 index 000000000..f9b3f43ec --- /dev/null +++ b/src/runtime/intern_.cs @@ -0,0 +1,48 @@ +using System; + +namespace Python.Runtime +{ + static class PyIdentifier + { + public static IntPtr __name__; + public static IntPtr __dict__; + public static IntPtr __doc__; + public static IntPtr __class__; + public static IntPtr __module__; + public static IntPtr __file__; + public static IntPtr __slots__; + public static IntPtr __self__; + public static IntPtr __annotations__; + public static IntPtr __init__; + public static IntPtr __repr__; + public static IntPtr __import__; + public static IntPtr __builtins__; + public static IntPtr builtins; + public static IntPtr __overloads__; + public static IntPtr Overloads; + } + + + static partial class InternString + { + private static readonly string[] _builtinNames = new string[] + { + "__name__", + "__dict__", + "__doc__", + "__class__", + "__module__", + "__file__", + "__slots__", + "__self__", + "__annotations__", + "__init__", + "__repr__", + "__import__", + "__builtins__", + "builtins", + "__overloads__", + "Overloads", + }; + } +} diff --git a/src/runtime/intern_.tt b/src/runtime/intern_.tt new file mode 100644 index 000000000..c7142ec9f --- /dev/null +++ b/src/runtime/intern_.tt @@ -0,0 +1,58 @@ +<#@ template debug="true" hostSpecific="true" #> +<#@ output extension=".cs" #> +<# + string[] internNames = new string[] + { + "__name__", + "__dict__", + "__doc__", + "__class__", + "__module__", + "__file__", + "__slots__", + "__self__", + "__annotations__", + + "__init__", + "__repr__", + "__import__", + "__builtins__", + + "builtins", + + "__overloads__", + "Overloads", + }; +#> +using System; + +namespace Python.Runtime +{ + static class PyIdentifier + { +<# + foreach (var name in internNames) + { +#> + public static IntPtr <#= name #>; +<# + } +#> + } + + + static partial class InternString + { + private static readonly string[] _builtinNames = new string[] + { +<# + foreach (var name in internNames) + { +#> + "<#= name #>", +<# + } +#> + }; + } +} diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..ff9a98622 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,6 +1,7 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Reflection; using System.Text; @@ -67,11 +68,72 @@ public ModulePropertyAttribute() } } + internal static partial class TypeOffset + { + public static int magic() => ManagedDataOffsets.Magic; + } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset + internal static class ManagedDataOffsets { - static ObjectOffset() + public static int Magic { get; internal set; } + public static readonly Dictionary NameMapping = new Dictionary(); + + static class DataOffsets + { + public static readonly int ob_data = 0; + public static readonly int ob_dict = 0; + + static DataOffsets() + { + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fields.Length; i++) + { + fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } + } + } + + static ManagedDataOffsets() + { + NameMapping = TypeOffset.GetOffsets(); + + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + size = fields.Length * IntPtr.Size; + } + + public static int GetSlotOffset(string name) + { + return NameMapping[name]; + } + + private static int BaseOffset(IntPtr type) + { + Debug.Assert(type != IntPtr.Zero); + int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); + Debug.Assert(typeSize > 0); + return typeSize; + } + + public static int DataOffset(IntPtr type) + { + return BaseOffset(type) + DataOffsets.ob_data; + } + + public static int DictOffset(IntPtr type) + { + return BaseOffset(type) + DataOffsets.ob_dict; + } + + public static int ob_data => DataOffsets.ob_data; + public static int ob_dict => DataOffsets.ob_dict; + public static int Size { get { return size; } } + + private static readonly int size; + } + + internal static class OriginalObjectOffsets + { + static OriginalObjectOffsets() { int size = IntPtr.Size; var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD @@ -82,42 +144,58 @@ static ObjectOffset() #endif ob_refcnt = (n + 0) * size; ob_type = (n + 1) * size; - ob_dict = (n + 2) * size; - ob_data = (n + 3) * size; } - public static int magic(IntPtr ob) + public static int Size { get { return size; } } + + private static readonly int size = +#if PYTHON_WITH_PYDEBUG + 4 * IntPtr.Size; +#else + 2 * IntPtr.Size; +#endif + +#if PYTHON_WITH_PYDEBUG + public static int _ob_next; + public static int _ob_prev; +#endif + public static int ob_refcnt; + public static int ob_type; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ObjectOffset + { + static ObjectOffset() + { +#if PYTHON_WITH_PYDEBUG + _ob_next = OriginalObjectOffsets._ob_next; + _ob_prev = OriginalObjectOffsets._ob_prev; +#endif + ob_refcnt = OriginalObjectOffsets.ob_refcnt; + ob_type = OriginalObjectOffsets.ob_type; + + size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + } + + public static int magic(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_data; - } - return ob_data; + return ManagedDataOffsets.DataOffset(type); } - public static int DictOffset(IntPtr ob) + public static int TypeDictOffset(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_dict; - } - return ob_dict; + return ManagedDataOffsets.DictOffset(type); } - public static int Size(IntPtr ob) + public static int Size(IntPtr pyType) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(pyType)) { return ExceptionOffset.Size(); } -#if PYTHON_WITH_PYDEBUG - return 6 * IntPtr.Size; -#else - return 4 * IntPtr.Size; -#endif + + return size; } #if PYTHON_WITH_PYDEBUG @@ -126,8 +204,15 @@ public static int Size(IntPtr ob) #endif public static int ob_refcnt; public static int ob_type; - private static int ob_dict; - private static int ob_data; + private static readonly int size; + + private static bool IsException(IntPtr pyObject) + { + var type = Runtime.PyObject_TYPE(pyObject); + return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) + || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -136,39 +221,30 @@ internal class ExceptionOffset static ExceptionOffset() { Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; + FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * size) + ObjectOffset.ob_type + size); + fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); } - } - public static int Size() - { - return ob_data + IntPtr.Size; + size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } + public static int Size() { return size; } + // PyException_HEAD // (start after PyObject_HEAD) public static int dict = 0; public static int args = 0; -#if PYTHON2 - public static int message = 0; -#elif PYTHON3 public static int traceback = 0; public static int context = 0; public static int cause = 0; public static int suppress_context = 0; -#endif - // extra c# data - public static int ob_dict; - public static int ob_data; + private static readonly int size; } -#if PYTHON3 [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] internal class BytesOffset { @@ -227,7 +303,7 @@ public static IntPtr AllocModuleDef(string modulename) byte[] ascii = Encoding.ASCII.GetBytes(modulename); int size = name + ascii.Length + 1; IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i < m_free; i += IntPtr.Size) + for (int i = 0; i <= m_free; i += IntPtr.Size) Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); @@ -259,7 +335,7 @@ public static void FreeModuleDef(IntPtr ptr) public static int name = 0; } -#endif // PYTHON3 + /// /// TypeFlags(): The actual bit values for the Type Flags stored @@ -269,61 +345,36 @@ public static void FreeModuleDef(IntPtr ptr) /// internal class TypeFlags { -#if PYTHON2 // these flags were removed in Python 3 - public static int HaveGetCharBuffer = (1 << 0); - public static int HaveSequenceIn = (1 << 1); - public static int GC = 0; - public static int HaveInPlaceOps = (1 << 3); - public static int CheckTypes = (1 << 4); - public static int HaveRichCompare = (1 << 5); - public static int HaveWeakRefs = (1 << 6); - public static int HaveIter = (1 << 7); - public static int HaveClass = (1 << 8); -#endif - public static int HeapType = (1 << 9); - public static int BaseType = (1 << 10); - public static int Ready = (1 << 12); - public static int Readying = (1 << 13); - public static int HaveGC = (1 << 14); + public const int HeapType = (1 << 9); + public const int BaseType = (1 << 10); + public const int Ready = (1 << 12); + public const int Readying = (1 << 13); + public const int HaveGC = (1 << 14); // 15 and 16 are reserved for stackless - public static int HaveStacklessExtension = 0; + public const int HaveStacklessExtension = 0; /* XXX Reusing reserved constants */ - public static int Managed = (1 << 15); // PythonNet specific - public static int Subclass = (1 << 16); // PythonNet specific - public static int HaveIndex = (1 << 17); + public const int Managed = (1 << 15); // PythonNet specific + public const int Subclass = (1 << 16); // PythonNet specific + public const int HaveIndex = (1 << 17); /* Objects support nb_index in PyNumberMethods */ - public static int HaveVersionTag = (1 << 18); - public static int ValidVersionTag = (1 << 19); - public static int IsAbstract = (1 << 20); - public static int HaveNewBuffer = (1 << 21); + public const int HaveVersionTag = (1 << 18); + public const int ValidVersionTag = (1 << 19); + public const int IsAbstract = (1 << 20); + public const int HaveNewBuffer = (1 << 21); // TODO: Implement FastSubclass functions - public static int IntSubclass = (1 << 23); - public static int LongSubclass = (1 << 24); - public static int ListSubclass = (1 << 25); - public static int TupleSubclass = (1 << 26); - public static int StringSubclass = (1 << 27); - public static int UnicodeSubclass = (1 << 28); - public static int DictSubclass = (1 << 29); - public static int BaseExceptionSubclass = (1 << 30); - public static int TypeSubclass = (1 << 31); - -#if PYTHON2 // Default flags for Python 2 - public static int Default = ( - HaveGetCharBuffer | - HaveSequenceIn | - HaveInPlaceOps | - HaveRichCompare | - HaveWeakRefs | - HaveIter | - HaveClass | - HaveStacklessExtension | - HaveIndex | - 0); -#elif PYTHON3 // Default flags for Python 3 - public static int Default = ( + public const int IntSubclass = (1 << 23); + public const int LongSubclass = (1 << 24); + public const int ListSubclass = (1 << 25); + public const int TupleSubclass = (1 << 26); + public const int StringSubclass = (1 << 27); + public const int UnicodeSubclass = (1 << 28); + public const int DictSubclass = (1 << 29); + public const int BaseExceptionSubclass = (1 << 30); + public const int TypeSubclass = (1 << 31); + + public const int Default = ( HaveStacklessExtension | HaveVersionTag); -#endif } @@ -334,7 +385,6 @@ internal class TypeFlags internal class Interop { - private static ArrayList keepAlive; private static Hashtable pmap; static Interop() @@ -351,8 +401,6 @@ static Interop() p[item.Name] = item; } - keepAlive = new ArrayList(); - Marshal.AllocHGlobal(IntPtr.Size); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -382,9 +430,6 @@ static Interop() pmap["nb_add"] = p["BinaryFunc"]; pmap["nb_subtract"] = p["BinaryFunc"]; pmap["nb_multiply"] = p["BinaryFunc"]; -#if PYTHON2 - pmap["nb_divide"] = p["BinaryFunc"]; -#endif pmap["nb_remainder"] = p["BinaryFunc"]; pmap["nb_divmod"] = p["BinaryFunc"]; pmap["nb_power"] = p["TernaryFunc"]; @@ -407,9 +452,6 @@ static Interop() pmap["nb_inplace_add"] = p["BinaryFunc"]; pmap["nb_inplace_subtract"] = p["BinaryFunc"]; pmap["nb_inplace_multiply"] = p["BinaryFunc"]; -#if PYTHON2 - pmap["nb_inplace_divide"] = p["BinaryFunc"]; -#endif pmap["nb_inplace_remainder"] = p["BinaryFunc"]; pmap["nb_inplace_power"] = p["TernaryFunc"]; pmap["nb_inplace_lshift"] = p["BinaryFunc"]; @@ -449,7 +491,10 @@ internal static Type GetPrototype(string name) return pmap[name] as Type; } - internal static IntPtr GetThunk(MethodInfo method, string funcType = null) + + internal static Dictionary allocatedThunks = new Dictionary(); + + internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { Type dt; if (funcType != null) @@ -457,20 +502,17 @@ internal static IntPtr GetThunk(MethodInfo method, string funcType = null) else dt = GetPrototype(method.Name); - if (dt != null) + if (dt == null) { - IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); - Delegate d = Delegate.CreateDelegate(dt, method); - Thunk cb = new Thunk(d); - Marshal.StructureToPtr(cb, tmp, false); - IntPtr fp = Marshal.ReadIntPtr(tmp, 0); - Marshal.FreeHGlobal(tmp); - keepAlive.Add(d); - return fp; + return ThunkInfo.Empty; } - return IntPtr.Zero; + Delegate d = Delegate.CreateDelegate(dt, method); + var info = new ThunkInfo(d); + allocatedThunks[info.Address] = d; + return info; } + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate IntPtr UnaryFunc(IntPtr ob); @@ -512,14 +554,46 @@ internal static IntPtr GetThunk(MethodInfo method, string funcType = null) } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal struct Thunk + internal class ThunkInfo { - public Delegate fn; + public readonly Delegate Target; + public readonly IntPtr Address; + + public static readonly ThunkInfo Empty = new ThunkInfo(null); - public Thunk(Delegate d) + public ThunkInfo(Delegate target) { - fn = d; + if (target == null) + { + return; + } + Target = target; + Address = Marshal.GetFunctionPointerForDelegate(target); } } + + [StructLayout(LayoutKind.Sequential)] + struct PyGC_Node + { + public IntPtr gc_next; + public IntPtr gc_prev; + public IntPtr gc_refs; + } + + [StructLayout(LayoutKind.Sequential)] + struct PyGC_Head + { + public PyGC_Node gc; + } + + + [StructLayout(LayoutKind.Sequential)] + struct PyMethodDef + { + public IntPtr ml_name; + public IntPtr ml_meth; + public int ml_flags; + public IntPtr ml_doc; + } + } diff --git a/src/runtime/interop27.cs b/src/runtime/interop27.cs deleted file mode 100644 index 4782e9d3b..000000000 --- a/src/runtime/interop27.cs +++ /dev/null @@ -1,150 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON27 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_compare = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_divide = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_nonzero = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_coerce = 0; - public static int nb_int = 0; - public static int nb_long = 0; - public static int nb_float = 0; - public static int nb_oct = 0; - public static int nb_hex = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_divide = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int sq_slice = 0; - public static int sq_ass_item = 0; - public static int sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getreadbuffer = 0; - public static int bf_getwritebuffer = 0; - public static int bf_getsegcount = 0; - public static int bf_getcharbuffer = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/interop33.cs b/src/runtime/interop33.cs deleted file mode 100644 index f684df6c6..000000000 --- a/src/runtime/interop33.cs +++ /dev/null @@ -1,143 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON33 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_reserved = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/interop34.cs b/src/runtime/interop34.cs deleted file mode 100644 index 6857ff2d0..000000000 --- a/src/runtime/interop34.cs +++ /dev/null @@ -1,144 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON34 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_reserved = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/interop35.cs b/src/runtime/interop35.cs deleted file mode 100644 index a30bfa4fd..000000000 --- a/src/runtime/interop35.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON35 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_as_async = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int am_await = 0; - public static int am_aiter = 0; - public static int am_anext = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int nb_matrix_multiply = 0; - public static int nb_inplace_matrix_multiply = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/interop36.cs b/src/runtime/interop36.cs index c46bcc2f5..4b3b8bfb9 100644 --- a/src/runtime/interop36.cs +++ b/src/runtime/interop36.cs @@ -1,149 +1,136 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. + +// Python 3.6: ABI flags: '' +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo -#if PYTHON36 using System; -using System.Collections; -using System.Collections.Specialized; +using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; + +using Python.Runtime.Native; namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset36 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset36() { } // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_as_async = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int am_await = 0; - public static int am_aiter = 0; - public static int am_anext = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int nb_matrix_multiply = 0; - public static int nb_inplace_matrix_multiply = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_print { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } } } - -#endif diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs new file mode 100644 index 000000000..951cb1068 --- /dev/null +++ b/src/runtime/interop37.cs @@ -0,0 +1,136 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.7: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset37 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset37() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_print { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + } +} diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs new file mode 100644 index 000000000..67a40eabd --- /dev/null +++ b/src/runtime/interop38.cs @@ -0,0 +1,138 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.8: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset38 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset38() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int tp_print { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + } +} diff --git a/src/runtime/interop39.cs b/src/runtime/interop39.cs new file mode 100644 index 000000000..cf3acc984 --- /dev/null +++ b/src/runtime/interop39.cs @@ -0,0 +1,138 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.9: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset39 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset39() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + } +} diff --git a/src/runtime/iterator.cs b/src/runtime/iterator.cs index c7c60ab19..089e8538a 100644 --- a/src/runtime/iterator.cs +++ b/src/runtime/iterator.cs @@ -10,10 +10,12 @@ namespace Python.Runtime internal class Iterator : ExtensionType { private IEnumerator iter; + private Type elemType; - public Iterator(IEnumerator e) + public Iterator(IEnumerator e, Type elemType) { iter = e; + this.elemType = elemType; } @@ -23,13 +25,25 @@ public Iterator(IEnumerator e) public static IntPtr tp_iternext(IntPtr ob) { var self = GetManagedObject(ob) as Iterator; - if (!self.iter.MoveNext()) + try { - Exceptions.SetError(Exceptions.StopIteration, Runtime.PyNone); + if (!self.iter.MoveNext()) + { + Exceptions.SetError(Exceptions.StopIteration, Runtime.PyNone); + return IntPtr.Zero; + } + } + catch (Exception e) + { + if (e.InnerException != null) + { + e = e.InnerException; + } + Exceptions.SetError(e); return IntPtr.Zero; } object item = self.iter.Current; - return Converter.ToPythonImplicit(item); + return Converter.ToPython(item, self.elemType); } public static IntPtr tp_iter(IntPtr ob) diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 9ee8d223b..b4baef835 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; +using System.Linq; namespace Python.Runtime { @@ -8,13 +11,72 @@ namespace Python.Runtime /// code. It defines the common fields that associate CLR and Python /// objects and common utilities to convert between those identities. /// + [Serializable] internal abstract class ManagedType { + internal enum TrackTypes + { + Untrack, + Extension, + Wrapper, + } + + [NonSerialized] internal GCHandle gcHandle; // Native handle + internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * + private static readonly Dictionary _managedObjs = new Dictionary(); + + internal void IncrRefCount() + { + Runtime.XIncref(pyHandle); + } + internal void DecrRefCount() + { + Runtime.XDecref(pyHandle); + } + + internal long RefCount + { + get + { + var gs = Runtime.PyGILState_Ensure(); + try + { + return Runtime.Refcount(pyHandle); + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + } + + internal GCHandle AllocGCHandle(TrackTypes track = TrackTypes.Untrack) + { + gcHandle = GCHandle.Alloc(this); + if (track != TrackTypes.Untrack) + { + _managedObjs.Add(this, track); + } + return gcHandle; + } + + internal void FreeGCHandle() + { + _managedObjs.Remove(this); + if (gcHandle.IsAllocated) + { + gcHandle.Free(); + gcHandle = default; + } + } + + internal static ManagedType GetManagedObject(BorrowedReference ob) + => GetManagedObject(ob.DangerousGetAddress()); /// /// Given a Python object, return the associated managed object or null. /// @@ -28,12 +90,16 @@ internal static ManagedType GetManagedObject(IntPtr ob) tp = ob; } - var flags = (int)Marshal.ReadIntPtr(tp, TypeOffset.tp_flags); + var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { IntPtr op = tp == ob ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(ob)); + : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); + if (op == IntPtr.Zero) + { + return null; + } var gc = (GCHandle)op; return (ManagedType)gc.Target; } @@ -41,6 +107,25 @@ internal static ManagedType GetManagedObject(IntPtr ob) return null; } + /// + /// Given a Python object, return the associated managed object type or null. + /// + internal static ManagedType GetManagedObjectType(IntPtr ob) + { + if (ob != IntPtr.Zero) + { + IntPtr tp = Runtime.PyObject_TYPE(ob); + var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + if ((flags & TypeFlags.Managed) != 0) + { + tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); + var gc = (GCHandle)tp; + return (ManagedType)gc.Target; + } + } + return null; + } + internal static ManagedType GetManagedObjectErr(IntPtr ob) { @@ -63,7 +148,7 @@ internal static bool IsManagedType(IntPtr ob) tp = ob; } - var flags = (int)Marshal.ReadIntPtr(tp, TypeOffset.tp_flags); + var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { return true; @@ -71,5 +156,109 @@ internal static bool IsManagedType(IntPtr ob) } return false; } + + public bool IsTypeObject() + { + return pyHandle == tpHandle; + } + + internal static IDictionary GetManagedObjects() + { + return _managedObjs; + } + + internal static void ClearTrackedObjects() + { + _managedObjs.Clear(); + } + + internal static int PyVisit(IntPtr ob, IntPtr visit, IntPtr arg) + { + if (ob == IntPtr.Zero) + { + return 0; + } + var visitFunc = NativeCall.GetDelegate(visit); + return visitFunc(ob, arg); + } + + /// + /// Wrapper for calling tp_clear + /// + internal void CallTypeClear() + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var clearPtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_clear); + if (clearPtr == IntPtr.Zero) + { + return; + } + var clearFunc = NativeCall.GetDelegate(clearPtr); + clearFunc(pyHandle); + } + + /// + /// Wrapper for calling tp_traverse + /// + internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var traversePtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_traverse); + if (traversePtr == IntPtr.Zero) + { + return; + } + var traverseFunc = NativeCall.GetDelegate(traversePtr); + + var visiPtr = Marshal.GetFunctionPointerForDelegate(visitproc); + traverseFunc(pyHandle, visiPtr, arg); + } + + protected void TypeClear() + { + ClearObjectDict(pyHandle); + } + + internal void Save(InterDomainContext context) + { + OnSave(context); + } + + internal void Load(InterDomainContext context) + { + OnLoad(context); + } + + protected virtual void OnSave(InterDomainContext context) { } + protected virtual void OnLoad(InterDomainContext context) { } + + protected static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + SetObjectDict(ob, IntPtr.Zero); + Runtime.XDecref(dict); + } + + protected static IntPtr GetObjectDict(IntPtr ob) + { + IntPtr type = Runtime.PyObject_TYPE(ob); + return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + } + + protected static void SetObjectDict(IntPtr ob, IntPtr value) + { + IntPtr type = Runtime.PyObject_TYPE(ob); + Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + } } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index bfb71e26d..36b406c7b 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,5 +1,8 @@ using System; +using System.Collections; +using System.IO; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -11,17 +14,55 @@ namespace Python.Runtime internal class MetaType : ManagedType { private static IntPtr PyCLRMetaType; + private static SlotsHolder _metaSlotsHodler; + internal static readonly string[] CustomMethods = new string[] + { + "__instancecheck__", + "__subclasscheck__", + }; /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// public static IntPtr Initialize() { - PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType)); + PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); return PyCLRMetaType; } + public static void Release() + { + if (Runtime.Refcount(PyCLRMetaType) > 1) + { + _metaSlotsHodler.ResetSlots(); + } + Runtime.Py_CLEAR(ref PyCLRMetaType); + _metaSlotsHodler = null; + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + Runtime.XIncref(PyCLRMetaType); + storage.PushValue(PyCLRMetaType); + } + + internal static IntPtr RestoreRuntimeData(RuntimeDataStorage storage) + { + PyCLRMetaType = storage.PopValue(); + _metaSlotsHodler = new SlotsHolder(PyCLRMetaType); + TypeManager.InitializeSlots(PyCLRMetaType, typeof(MetaType), _metaSlotsHodler); + + IntPtr mdef = Marshal.ReadIntPtr(PyCLRMetaType, TypeOffset.tp_methods); + foreach (var methodName in CustomMethods) + { + var mi = typeof(MetaType).GetMethod(methodName); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + _metaSlotsHodler.KeeapAlive(thunkInfo); + mdef = TypeManager.WriteMethodDef(mdef, methodName, thunkInfo.Address); + } + return PyCLRMetaType; + } /// /// Metatype __new__ implementation. This is called to create a new @@ -29,7 +70,7 @@ public static IntPtr Initialize() /// public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { - int len = Runtime.PyTuple_Size(args); + var len = Runtime.PyTuple_Size(args); if (len < 3) { return Exceptions.RaiseTypeError("invalid argument list"); @@ -63,13 +104,20 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) var cb = GetManagedObject(base_type) as ClassBase; if (cb != null) { - if (!cb.CanSubclass()) + try + { + if (!cb.CanSubclass()) + { + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + } + } + catch (SerializationException) { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); } } - IntPtr slots = Runtime.PyDict_GetItemString(dict, "__slots__"); + IntPtr slots = Runtime.PyDict_GetItem(dict, PyIdentifier.__slots__); if (slots != IntPtr.Zero) { return Exceptions.RaiseTypeError("subclasses of managed classes do not support __slots__"); @@ -105,7 +153,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) flags |= TypeFlags.BaseType; flags |= TypeFlags.Subclass; flags |= TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + Util.WriteCLong(type, TypeOffset.tp_flags, flags); TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); @@ -157,23 +205,13 @@ public static IntPtr tp_call(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - IntPtr py__init__ = Runtime.PyString_FromString("__init__"); - IntPtr type = Runtime.PyObject_TYPE(obj); - IntPtr init = Runtime._PyType_Lookup(type, py__init__); - Runtime.XDecref(py__init__); + var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__); Runtime.PyErr_Clear(); if (init != IntPtr.Zero) { - IntPtr bound = Runtime.GetBoundArgTuple(obj, args); - if (bound == IntPtr.Zero) - { - Runtime.XDecref(obj); - return IntPtr.Zero; - } - - IntPtr result = Runtime.PyObject_Call(init, bound, kw); - Runtime.XDecref(bound); + IntPtr result = Runtime.PyObject_Call(init, args, kw); + Runtime.XDecref(init); if (result == IntPtr.Zero) { @@ -211,7 +249,7 @@ public static int tp_setattro(IntPtr tp, IntPtr name, IntPtr value) IntPtr fp = Marshal.ReadIntPtr(dt, TypeOffset.tp_descr_set); if (fp != IntPtr.Zero) { - return NativeCall.Impl.Int_Call_3(fp, descr, name, value); + return NativeCall.Int_Call_3(fp, descr, name, value); } Exceptions.SetError(Exceptions.AttributeError, "attribute is read-only"); return -1; @@ -247,7 +285,7 @@ public static void tp_dealloc(IntPtr tp) { // Fix this when we dont cheat on the handle for subclasses! - var flags = (int)Marshal.ReadIntPtr(tp, TypeOffset.tp_flags); + var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); @@ -270,12 +308,13 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) { var cb = GetManagedObject(tp) as ClassBase; - if (cb == null) + if (cb == null || !cb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; } + Runtime.XIncref(args); using (var argsObj = new PyList(args)) { if (argsObj.Length() != 1) @@ -301,13 +340,13 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) } var otherCb = GetManagedObject(otherType.Handle) as ClassBase; - if (otherCb == null) + if (otherCb == null || !otherCb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; } - return Converter.ToPython(cb.type.IsAssignableFrom(otherCb.type)); + return Converter.ToPython(cb.type.Value.IsAssignableFrom(otherCb.type.Value)); } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index f0c58f34f..5de0ecc00 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -1,30 +1,40 @@ using System; using System.Collections; using System.Reflection; +using System.Text; +using System.Collections.Generic; +using System.Linq; namespace Python.Runtime { + using MaybeMethodBase = MaybeMethodBase; /// /// A MethodBinder encapsulates information about a (possibly overloaded) /// managed method, and is responsible for selecting the right method given /// a set of Python arguments. This is also used as a base class for the /// ConstructorBinder, a minor variation used to invoke constructors. /// + [Serializable] internal class MethodBinder { - public ArrayList list; + public List list; + + [NonSerialized] public MethodBase[] methods; + + [NonSerialized] public bool init = false; - public bool allow_threads = true; + public const bool DefaultAllowThreads = true; + public bool allow_threads = DefaultAllowThreads; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MaybeMethodBase(mi) }; } public int Count @@ -160,7 +170,7 @@ internal MethodBase[] GetMethods() { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); + methods = (from method in list where method.Valid select method.Value).ToArray(); init = true; } return methods; @@ -176,6 +186,11 @@ internal MethodBase[] GetMethods() /// internal static int GetPrecedence(MethodBase mi) { + if (mi == null) + { + return int.MaxValue; + } + ParameterInfo[] pi = mi.GetParameters(); int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -200,6 +215,16 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsArray) + { + Type e = t.GetElementType(); + if (e == objectType) + { + return 2500; + } + return 100 + ArgPrecedence(e); + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) @@ -247,16 +272,6 @@ internal static int ArgPrecedence(Type t) return 40; } - if (t.IsArray) - { - Type e = t.GetElementType(); - if (e == objectType) - { - return 2500; - } - return 100 + ArgPrecedence(e); - } - return 2000; } @@ -275,14 +290,45 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) return Bind(inst, args, kw, info, null); } + private readonly struct MatchedMethod { + public MatchedMethod(int kwargsMatched, int defaultsNeeded, object[] margs, int outs, MethodBase mb) + { + KwargsMatched = kwargsMatched; + DefaultsNeeded = defaultsNeeded; + ManagedArgs = margs; + Outs = outs; + Method = mb; + } + + public int KwargsMatched { get; } + public int DefaultsNeeded { get; } + public object[] ManagedArgs { get; } + public int Outs { get; } + public MethodBase Method { get; } + } + internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { // loop to find match, return invoker w/ or /wo error MethodBase[] _methods = null; - int pynargs = Runtime.PyTuple_Size(args); - object arg; + + var kwargDict = new Dictionary(); + if (kw != IntPtr.Zero) + { + var pynkwargs = (int)Runtime.PyDict_Size(kw); + IntPtr keylist = Runtime.PyDict_Keys(kw); + IntPtr valueList = Runtime.PyDict_Values(kw); + for (int i = 0; i < pynkwargs; ++i) + { + var keyStr = Runtime.GetManagedString(Runtime.PyList_GetItem(new BorrowedReference(keylist), i)); + kwargDict[keyStr] = Runtime.PyList_GetItem(new BorrowedReference(valueList), i).DangerousGetAddress(); + } + Runtime.XDecref(keylist); + Runtime.XDecref(valueList); + } + + var pynargs = (int)Runtime.PyTuple_Size(args); var isGeneric = false; - ArrayList defaultArgList = null; if (info != null) { _methods = new MethodBase[1]; @@ -292,7 +338,9 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { _methods = GetMethods(); } - Type clrtype; + + var argMatchedMethods = new List(_methods.Length); + // TODO: Clean up foreach (MethodBase mi in _methods) { @@ -301,192 +349,418 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth isGeneric = true; } ParameterInfo[] pi = mi.GetParameters(); - int clrnargs = pi.Length; - var match = false; - int arrayStart = -1; + ArrayList defaultArgList; + bool paramsArray; + int kwargsMatched; + int defaultsNeeded; + bool isOperator = OperatorMethod.IsOperatorMethod(mi); + // Binary operator methods will have 2 CLR args but only one Python arg + // (unary operators will have 1 less each), since Python operator methods are bound. + isOperator = isOperator && pynargs == pi.Length - 1; + bool isReverse = isOperator && OperatorMethod.IsReverse((MethodInfo)mi); // Only cast if isOperator. + if (isReverse && OperatorMethod.IsComparisonOp((MethodInfo)mi)) + continue; // Comparison operators in Python have no reverse mode. + if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList, out kwargsMatched, out defaultsNeeded) && !isOperator) + { + continue; + } + // Preprocessing pi to remove either the first or second argument. + if (isOperator && !isReverse) { + // The first Python arg is the right operand, while the bound instance is the left. + // We need to skip the first (left operand) CLR argument. + pi = pi.Skip(1).ToArray(); + } + else if (isOperator && isReverse) { + // The first Python arg is the left operand. + // We need to take the first CLR argument. + pi = pi.Take(1).ToArray(); + } var outs = 0; - - if (pynargs == clrnargs) + var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, + needsResolution: _methods.Length > 1, // If there's more than one possible match. + outs: out outs); + if (margs == null) { - match = true; + continue; } - else if (pynargs < clrnargs) + if (isOperator) { - match = true; - defaultArgList = new ArrayList(); - for (int v = pynargs; v < clrnargs; v++) + if (inst != IntPtr.Zero) { - if (pi[v].DefaultValue == DBNull.Value) + if (ManagedType.GetManagedObject(inst) is CLRObject co) { - match = false; - } - else - { - defaultArgList.Add(pi[v].DefaultValue); + bool isUnary = pynargs == 0; + // Postprocessing to extend margs. + var margsTemp = isUnary ? new object[1] : new object[2]; + // If reverse, the bound instance is the right operand. + int boundOperandIndex = isReverse ? 1 : 0; + // If reverse, the passed instance is the left operand. + int passedOperandIndex = isReverse ? 0 : 1; + margsTemp[boundOperandIndex] = co.inst; + if (!isUnary) + { + margsTemp[passedOperandIndex] = margs[0]; + } + margs = margsTemp; } + else { break; } } } - else if (pynargs > clrnargs && clrnargs > 0 && - Attribute.IsDefined(pi[clrnargs - 1], typeof(ParamArrayAttribute))) + + + var matchedMethod = new MatchedMethod(kwargsMatched, defaultsNeeded, margs, outs, mi); + argMatchedMethods.Add(matchedMethod); + } + if (argMatchedMethods.Count > 0) + { + var bestKwargMatchCount = argMatchedMethods.Max(x => x.KwargsMatched); + var fewestDefaultsRequired = argMatchedMethods.Where(x => x.KwargsMatched == bestKwargMatchCount).Min(x => x.DefaultsNeeded); + + int bestCount = 0; + int bestMatchIndex = -1; + + for (int index = 0; index < argMatchedMethods.Count; index++) { - // This is a `foo(params object[] bar)` style method - match = true; - arrayStart = clrnargs - 1; + var testMatch = argMatchedMethods[index]; + if (testMatch.DefaultsNeeded == fewestDefaultsRequired && testMatch.KwargsMatched == bestKwargMatchCount) + { + bestCount++; + if (bestMatchIndex == -1) + bestMatchIndex = index; + } } - if (match) + if (bestCount > 1 && fewestDefaultsRequired > 0) { - var margs = new object[clrnargs]; + // Best effort for determining method to match on gives multiple possible + // matches and we need at least one default argument - bail from this point + return null; + } - for (var n = 0; n < clrnargs; n++) + // If we're here either: + // (a) There is only one best match + // (b) There are multiple best matches but none of them require + // default arguments + // in the case of (a) we're done by default. For (b) regardless of which + // method we choose, all arguments are specified _and_ can be converted + // from python to C# so picking any will suffice + MatchedMethod bestMatch = argMatchedMethods[bestMatchIndex]; + var margs = bestMatch.ManagedArgs; + var outs = bestMatch.Outs; + var mi = bestMatch.Method; + + object target = null; + if (!mi.IsStatic && inst != IntPtr.Zero) + { + //CLRObject co = (CLRObject)ManagedType.GetManagedObject(inst); + // InvalidCastException: Unable to cast object of type + // 'Python.Runtime.ClassObject' to type 'Python.Runtime.CLRObject' + var co = ManagedType.GetManagedObject(inst) as CLRObject; + + // Sanity check: this ensures a graceful exit if someone does + // something intentionally wrong like call a non-static method + // on the class rather than on an instance of the class. + // XXX maybe better to do this before all the other rigmarole. + if (co == null) { - IntPtr op; - if (n < pynargs) - { - if (arrayStart == n) - { - // map remaining Python arguments to a tuple since - // the managed function accepts it - hopefully :] - op = Runtime.PyTuple_GetSlice(args, arrayStart, pynargs); - } - else - { - op = Runtime.PyTuple_GetItem(args, n); - } + return null; + } + target = co.inst; + } - // this logic below handles cases when multiple overloading methods - // are ambiguous, hence comparison between Python and CLR types - // is necessary - clrtype = null; - IntPtr pyoptype; - if (_methods.Length > 1) - { - pyoptype = IntPtr.Zero; - pyoptype = Runtime.PyObject_Type(op); - Exceptions.Clear(); - if (pyoptype != IntPtr.Zero) - { - clrtype = Converter.GetTypeByAlias(pyoptype); - } - Runtime.XDecref(pyoptype); - } + return new Binding(mi, target, margs, outs); + } + // We weren't able to find a matching method but at least one + // is a generic method and info is null. That happens when a generic + // method was not called using the [] syntax. Let's introspect the + // type of the arguments and use it to construct the correct method. + if (isGeneric && info == null && methodinfo != null) + { + Type[] types = Runtime.PythonArgsToTypeArray(args, true); + MethodInfo mi = MatchParameters(methodinfo, types); + return Bind(inst, args, kw, mi, null); + } + return null; + } + static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out bool isNewReference) + { + isNewReference = false; + IntPtr op; + // for a params method, we may have a sequence or single/multiple items + // here we look to see if the item at the paramIndex is there or not + // and then if it is a sequence itself. + if ((pyArgCount - arrayStart) == 1) + { + // we only have one argument left, so we need to check it + // to see if it is a sequence or a single item + IntPtr item = Runtime.PyTuple_GetItem(args, arrayStart); + if (!Runtime.PyString_Check(item) && Runtime.PySequence_Check(item)) + { + // it's a sequence (and not a string), so we use it as the op + op = item; + } + else + { + isNewReference = true; + op = Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount); + } + } + else + { + isNewReference = true; + op = Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount); + } + return op; + } - if (clrtype != null) - { - var typematch = false; - if ((pi[n].ParameterType != typeof(object)) && (pi[n].ParameterType != clrtype)) - { - IntPtr pytype = Converter.GetPythonTypeByAlias(pi[n].ParameterType); - pyoptype = Runtime.PyObject_Type(op); - Exceptions.Clear(); - if (pyoptype != IntPtr.Zero) - { - if (pytype != pyoptype) - { - typematch = false; - } - else - { - typematch = true; - clrtype = pi[n].ParameterType; - } - } - if (!typematch) - { - // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); - TypeCode paramtypecode = Type.GetTypeCode(clrtype); - if (argtypecode == paramtypecode) - { - typematch = true; - clrtype = pi[n].ParameterType; - } - } - Runtime.XDecref(pyoptype); - if (!typematch) - { - margs = null; - break; - } - } - else - { - typematch = true; - clrtype = pi[n].ParameterType; - } - } - else - { - clrtype = pi[n].ParameterType; - } + /// + /// Attempts to convert Python positional argument tuple and keyword argument table + /// into an array of managed objects, that can be passed to a method. + /// + /// Information about expected parameters + /// true, if the last parameter is a params array. + /// A pointer to the Python argument tuple + /// Number of arguments, passed by Python + /// Dictionary of keyword argument name to python object pointer + /// A list of default values for omitted parameters + /// true, if overloading resolution is required + /// Returns number of output parameters + /// An array of .NET arguments, that can be passed to a method. + static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, + IntPtr args, int pyArgCount, + Dictionary kwargDict, + ArrayList defaultArgList, + bool needsResolution, + out int outs) + { + outs = 0; + var margs = new object[pi.Length]; + int arrayStart = paramsArray ? pi.Length - 1 : -1; - if (pi[n].IsOut || clrtype.IsByRef) - { - outs++; - } + for (int paramIndex = 0; paramIndex < pi.Length; paramIndex++) + { + var parameter = pi[paramIndex]; + bool hasNamedParam = kwargDict.ContainsKey(parameter.Name); + bool isNewReference = false; - if (!Converter.ToManaged(op, clrtype, out arg, false)) - { - Exceptions.Clear(); - margs = null; - break; - } - if (arrayStart == n) - { - // GetSlice() creates a new reference but GetItem() - // returns only a borrow reference. - Runtime.XDecref(op); - } - margs[n] = arg; - } - else - { - if (defaultArgList != null) - { - margs[n] = defaultArgList[n - pynargs]; - } - } + if (paramIndex >= pyArgCount && !(hasNamedParam || (paramsArray && paramIndex == arrayStart))) + { + if (defaultArgList != null) + { + margs[paramIndex] = defaultArgList[paramIndex - pyArgCount]; } - if (margs == null) + continue; + } + + IntPtr op; + if (hasNamedParam) + { + op = kwargDict[parameter.Name]; + } + else + { + if(arrayStart == paramIndex) + { + op = HandleParamsArray(args, arrayStart, pyArgCount, out isNewReference); + } + else { - continue; + op = Runtime.PyTuple_GetItem(args, paramIndex); } + } + + bool isOut; + if (!TryConvertArgument(op, parameter.ParameterType, needsResolution, out margs[paramIndex], out isOut)) + { + return null; + } + + if (isNewReference) + { + // TODO: is this a bug? Should this happen even if the conversion fails? + // GetSlice() creates a new reference but GetItem() + // returns only a borrow reference. + Runtime.XDecref(op); + } + + if (isOut) + { + outs++; + } + } + + return margs; + } - object target = null; - if (!mi.IsStatic && inst != IntPtr.Zero) + /// + /// Try to convert a Python argument object to a managed CLR type. + /// + /// Pointer to the object at a particular parameter. + /// That parameter's managed type. + /// There are multiple overloading methods that need resolution. + /// Converted argument. + /// Whether the CLR type is passed by reference. + /// + static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResolution, + out object arg, out bool isOut) + { + arg = null; + isOut = false; + var clrtype = TryComputeClrArgumentType(parameterType, op, needsResolution: needsResolution); + if (clrtype == null) + { + return false; + } + + if (!Converter.ToManaged(op, clrtype, out arg, false)) + { + Exceptions.Clear(); + return false; + } + + isOut = clrtype.IsByRef; + return true; + } + + static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool needsResolution) + { + // this logic below handles cases when multiple overloading methods + // are ambiguous, hence comparison between Python and CLR types + // is necessary + Type clrtype = null; + IntPtr pyoptype; + if (needsResolution) + { + // HACK: each overload should be weighted in some way instead + pyoptype = Runtime.PyObject_Type(argument); + Exceptions.Clear(); + if (pyoptype != IntPtr.Zero) + { + clrtype = Converter.GetTypeByAlias(pyoptype); + } + Runtime.XDecref(pyoptype); + } + + if (clrtype != null) + { + var typematch = false; + if ((parameterType != typeof(object)) && (parameterType != clrtype)) + { + IntPtr pytype = Converter.GetPythonTypeByAlias(parameterType); + pyoptype = Runtime.PyObject_Type(argument); + Exceptions.Clear(); + if (pyoptype != IntPtr.Zero) + { + if (pytype != pyoptype) + { + typematch = false; + } + else + { + typematch = true; + clrtype = parameterType; + } + } + if (!typematch) { - //CLRObject co = (CLRObject)ManagedType.GetManagedObject(inst); - // InvalidCastException: Unable to cast object of type - // 'Python.Runtime.ClassObject' to type 'Python.Runtime.CLRObject' - var co = ManagedType.GetManagedObject(inst) as CLRObject; - - // Sanity check: this ensures a graceful exit if someone does - // something intentionally wrong like call a non-static method - // on the class rather than on an instance of the class. - // XXX maybe better to do this before all the other rigmarole. - if (co == null) + // this takes care of enum values + TypeCode argtypecode = Type.GetTypeCode(parameterType); + TypeCode paramtypecode = Type.GetTypeCode(clrtype); + if (argtypecode == paramtypecode) { - return null; + typematch = true; + clrtype = parameterType; } - target = co.inst; } + Runtime.XDecref(pyoptype); + if (!typematch) + { + return null; + } + } + else + { + typematch = true; + clrtype = parameterType; + } + } + else + { + clrtype = parameterType; + } - return new Binding(mi, target, margs, outs); + return clrtype; + } + /// + /// Check whether the number of Python and .NET arguments match, and compute additional arg information. + /// + /// Number of positional args passed from Python. + /// Parameters of the specified .NET method. + /// Keyword args passed from Python. + /// True if the final param of the .NET method is an array (`params` keyword). + /// List of default values for arguments. + /// Number of kwargs from Python that are also present in the .NET method. + /// Number of non-null defaultsArgs. + /// + static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] parameters, + Dictionary kwargDict, + out bool paramsArray, + out ArrayList defaultArgList, + out int kwargsMatched, + out int defaultsNeeded) + { + defaultArgList = null; + var match = false; + paramsArray = parameters.Length > 0 ? Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute)) : false; + kwargsMatched = 0; + defaultsNeeded = 0; + if (positionalArgumentCount == parameters.Length && kwargDict.Count == 0) + { + match = true; + } + else if (positionalArgumentCount < parameters.Length && (!paramsArray || positionalArgumentCount == parameters.Length - 1)) + { + match = true; + // every parameter past 'positionalArgumentCount' must have either + // a corresponding keyword arg or a default param, unless the method + // method accepts a params array (which cannot have a default value) + defaultArgList = new ArrayList(); + for (var v = positionalArgumentCount; v < parameters.Length; v++) + { + if (kwargDict.ContainsKey(parameters[v].Name)) + { + // we have a keyword argument for this parameter, + // no need to check for a default parameter, but put a null + // placeholder in defaultArgList + defaultArgList.Add(null); + kwargsMatched++; + } + else if (parameters[v].IsOptional) + { + // IsOptional will be true if the parameter has a default value, + // or if the parameter has the [Optional] attribute specified. + // The GetDefaultValue() extension method will return the value + // to be passed in as the parameter value + defaultArgList.Add(parameters[v].GetDefaultValue()); + defaultsNeeded++; + } + else if (!paramsArray) + { + match = false; + } } } - // We weren't able to find a matching method but at least one - // is a generic method and info is null. That happens when a generic - // method was not called using the [] syntax. Let's introspect the - // type of the arguments and use it to construct the correct method. - if (isGeneric && info == null && methodinfo != null) + else if (positionalArgumentCount > parameters.Length && parameters.Length > 0 && + Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute))) { - Type[] types = Runtime.PythonArgsToTypeArray(args, true); - MethodInfo mi = MatchParameters(methodinfo, types); - return Bind(inst, args, kw, mi, null); + // This is a `foo(params object[] bar)` style method + match = true; + paramsArray = true; } - return null; + + return match; } internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) @@ -499,15 +773,68 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i return Invoke(inst, args, kw, info, null); } + protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) + { + long argCount = Runtime.PyTuple_Size(args); + to.Append("("); + for (long argIndex = 0; argIndex < argCount; argIndex++) + { + var arg = Runtime.PyTuple_GetItem(args, argIndex); + if (arg != IntPtr.Zero) + { + var type = Runtime.PyObject_Type(arg); + if (type != IntPtr.Zero) + { + try + { + var description = Runtime.PyObject_Unicode(type); + if (description != IntPtr.Zero) + { + to.Append(Runtime.GetManagedString(description)); + Runtime.XDecref(description); + } + } + finally + { + Runtime.XDecref(type); + } + } + } + + if (argIndex + 1 < argCount) + to.Append(", "); + } + to.Append(')'); + } + internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { + // No valid methods, nothing to bind. + if (GetMethods().Length == 0) + { + var msg = new StringBuilder("The underlying C# method(s) have been deleted"); + if (list.Count > 0 && list[0].Name != null) + { + msg.Append($": {list[0].ToString()}"); + } + return Exceptions.RaiseTypeError(msg.ToString());; + } + Binding binding = Bind(inst, args, kw, info, methodinfo); object result; IntPtr ts = IntPtr.Zero; if (binding == null) { - Exceptions.SetError(Exceptions.TypeError, "No method matches given arguments"); + var value = new StringBuilder("No method matches given arguments"); + if (methodinfo != null && methodinfo.Length > 0) + { + value.Append($" for {methodinfo[0].Name}"); + } + + value.Append(": "); + AppendArgumentTypes(to: value, args); + Exceptions.SetError(Exceptions.TypeError, value.ToString()); return IntPtr.Zero; } @@ -567,7 +894,7 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i Type pt = pi[i].ParameterType; if (pi[i].IsOut || pt.IsByRef) { - v = Converter.ToPython(binding.args[i], pt); + v = Converter.ToPython(binding.args[i], pt.GetElementType()); Runtime.PyTuple_SetItem(t, n, v); n++; } @@ -592,12 +919,38 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i /// /// Utility class to sort method info by parameter type precedence. /// - internal class MethodSorter : IComparer + internal class MethodSorter : IComparer { - int IComparer.Compare(object m1, object m2) + int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); + MethodBase me1 = m1.UnsafeValue; + MethodBase me2 = m2.UnsafeValue; + if (me1 == null && me2 == null) + { + return 0; + } + else if (me1 == null) + { + return -1; + } + else if (me2 == null) + { + return 1; + } + + if (me1.DeclaringType != me2.DeclaringType) + { + // m2's type derives from m1's type, favor m2 + if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) + return 1; + + // m1's type derives from m2's type, favor m1 + if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) + return -1; + } + + int p1 = MethodBinder.GetPrecedence(me1); + int p2 = MethodBinder.GetPrecedence(me2); if (p1 < p2) { return -1; @@ -631,4 +984,33 @@ internal Binding(MethodBase info, object inst, object[] args, int outs) this.outs = outs; } } + + + static internal class ParameterInfoExtensions + { + public static object GetDefaultValue(this ParameterInfo parameterInfo) + { + // parameterInfo.HasDefaultValue is preferable but doesn't exist in .NET 4.0 + bool hasDefaultValue = (parameterInfo.Attributes & ParameterAttributes.HasDefault) == + ParameterAttributes.HasDefault; + + if (hasDefaultValue) + { + return parameterInfo.DefaultValue; + } + else + { + // [OptionalAttribute] was specified for the parameter. + // See https://stackoverflow.com/questions/3416216/optionalattribute-parameters-default-value + // for rules on determining the value to pass to the parameter + var type = parameterInfo.ParameterType; + if (type == typeof(object)) + return Type.Missing; + else if (type.IsValueType) + return Activator.CreateInstance(type); + else + return null; + } + } + } } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 07090a92c..46b62807d 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -4,14 +4,16 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python binding type for CLR methods. These work much like /// standard Python method bindings, but the same type is used to bind /// both static and instance methods. /// + [Serializable] internal class MethodBinding : ExtensionType { - internal MethodInfo info; + internal MaybeMethodInfo info; internal MethodObject m; internal IntPtr target; internal IntPtr targetType; @@ -21,11 +23,15 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) Runtime.XIncref(target); this.target = target; - Runtime.XIncref(targetType); if (targetType == IntPtr.Zero) { targetType = Runtime.PyObject_Type(target); } + else + { + Runtime.XIncref(targetType); + } + this.targetType = targetType; this.info = null; @@ -36,6 +42,12 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } + private void ClearMembers() + { + Runtime.Py_CLEAR(ref target); + Runtime.Py_CLEAR(ref targetType); + } + /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -56,7 +68,6 @@ public static IntPtr mp_subscript(IntPtr tp, IntPtr idx) } var mb = new MethodBinding(self.m, self.target) { info = mi }; - Runtime.XIncref(mb.pyHandle); return mb.pyHandle; } @@ -74,7 +85,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return IntPtr.Zero; } - string name = Runtime.GetManagedString(key); + string name = InternString.GetManagedString(key); switch (name) { case "__doc__": @@ -85,7 +96,6 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) case "__overloads__": case "Overloads": var om = new OverloadMapper(self.m, self.target); - Runtime.XIncref(om.pyHandle); return om.pyHandle; default: return Runtime.PyObject_GenericGetAttr(ob, key); @@ -102,15 +112,16 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) // This works around a situation where the wrong generic method is picked, // for example this method in the tests: string Overloaded(int arg1, int arg2, string arg3) - if (self.info != null) + if (self.info.Valid) { - if (self.info.IsGenericMethod) + var info = self.info.Value; + if (info.IsGenericMethod) { - int len = Runtime.PyTuple_Size(args); //FIXME: Never used + var len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { - Type[] genericTp = self.info.GetGenericArguments(); + Type[] genericTp = info.GetGenericArguments(); MethodInfo betterMatch = MethodBinder.MatchSignatureAndParameters(self.m.info, genericTp, sigTp); if (betterMatch != null) { @@ -131,7 +142,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (target == IntPtr.Zero && !self.m.IsStatic()) { - int len = Runtime.PyTuple_Size(args); + var len = Runtime.PyTuple_Size(args); if (len < 1) { Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); @@ -155,9 +166,9 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (inst?.inst is IPythonDerivedType) { var baseType = GetManagedObject(self.targetType) as ClassBase; - if (baseType != null) + if (baseType != null && baseType.type.Valid) { - string baseMethodName = "_" + baseType.type.Name + "__" + self.m.name; + string baseMethodName = "_" + baseType.type.Value.Name + "__" + self.m.name; IntPtr baseMethod = Runtime.PyObject_GetAttrString(target, baseMethodName); if (baseMethod != IntPtr.Zero) { @@ -175,8 +186,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) } } } - - return self.m.Invoke(target, args, kw, self.info); + return self.m.Invoke(target, args, kw, self.info.UnsafeValue); } finally { @@ -239,9 +249,22 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - Runtime.XDecref(self.targetType); - FinalizeObject(self); + self.ClearMembers(); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (MethodBinding)GetManagedObject(ob); + self.ClearMembers(); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + Runtime.XIncref(target); + Runtime.XIncref(targetType); } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..37c01f5c5 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -1,8 +1,12 @@ using System; +using System.Collections.Generic; using System.Reflection; +using System.Linq; namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethodBase; + /// /// Implements a Python type that represents a CLR method. Method objects /// support a subscript syntax [] to allow explicit overload selection. @@ -10,41 +14,48 @@ namespace Python.Runtime /// /// TODO: ForbidPythonThreadsAttribute per method info /// + [Serializable] internal class MethodObject : ExtensionType { - internal MethodInfo[] info; + [NonSerialized] + private MethodInfo[] _info = null; + private readonly List infoList; internal string name; internal MethodBinding unbound; - internal MethodBinder binder; + internal readonly MethodBinder binder; internal bool is_static = false; + internal IntPtr doc; internal Type type; - public MethodObject(Type type, string name, MethodInfo[] info) - { - _MethodObject(type, name, info); - } - - public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads) - { - _MethodObject(type, name, info); - binder.allow_threads = allow_threads; - } - - private void _MethodObject(Type type, string name, MethodInfo[] info) + public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = MethodBinder.DefaultAllowThreads) { this.type = type; this.name = name; - this.info = info; + this.infoList = new List(); binder = new MethodBinder(); foreach (MethodInfo item in info) { + this.infoList.Add(item); binder.AddMethod(item); if (item.IsStatic) { this.is_static = true; } } + binder.allow_threads = allow_threads; + } + + internal MethodInfo[] info + { + get + { + if (_info == null) + { + _info = (from i in infoList where i.Valid select i.Value).ToArray(); + } + return _info; + } } public virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) @@ -109,6 +120,16 @@ internal bool IsStatic() return is_static; } + private void ClearMembers() + { + Runtime.Py_CLEAR(ref doc); + if (unbound != null) + { + Runtime.XDecref(unbound.pyHandle); + unbound = null; + } + } + /// /// Descriptor __getattribute__ implementation. /// @@ -121,8 +142,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return Exceptions.RaiseTypeError("string expected"); } - string name = Runtime.GetManagedString(key); - if (name == "__doc__") + if (Runtime.PyUnicode_Compare(key, PyIdentifier.__doc__) == 0) { IntPtr doc = self.GetDocString(); Runtime.XIncref(doc); @@ -196,12 +216,27 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); - Runtime.XDecref(self.doc); - if (self.unbound != null) + self.ClearMembers(); + ClearObjectDict(ob); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (MethodObject)GetManagedObject(ob); + self.ClearMembers(); + ClearObjectDict(ob); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + if (unbound != null) { - Runtime.XDecref(self.unbound.pyHandle); + Runtime.XIncref(unbound.pyHandle); } - ExtensionType.FinalizeObject(self); + Runtime.XIncref(doc); } } } diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index 2f3ce3ef2..4caefccb6 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -12,24 +12,45 @@ internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + private ThunkInfo _thunk; + + private bool _disposed = false; + public MethodWrapper(Type type, string name, string funcType = null) { // Turn the managed method into a function pointer - IntPtr fp = Interop.GetThunk(type.GetMethod(name), funcType); + _thunk = Interop.GetThunk(type.GetMethod(name), funcType); // Allocate and initialize a PyMethodDef structure to represent // the managed method, then create a PyCFunction. mdef = Runtime.PyMem_Malloc(4 * IntPtr.Size); - TypeManager.WriteMethodDef(mdef, name, fp, 0x0003); + TypeManager.WriteMethodDef(mdef, name, _thunk.Address, 0x0003); ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } + public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); } + + public void Release() + { + if (_disposed) + { + return; + } + _disposed = true; + bool freeDef = Runtime.Refcount(ptr) == 1; + Runtime.XDecref(ptr); + if (freeDef && mdef != IntPtr.Zero) + { + Runtime.PyMem_Free(mdef); + mdef = IntPtr.Zero; + } + } } } diff --git a/src/runtime/modulefunctionobject.cs b/src/runtime/modulefunctionobject.cs index 8f8692af9..e7a2c515a 100644 --- a/src/runtime/modulefunctionobject.cs +++ b/src/runtime/modulefunctionobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Module level functions /// + [Serializable] internal class ModuleFunctionObject : MethodObject { public ModuleFunctionObject(Type type, string name, MethodInfo[] info, bool allow_threads) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 258d77bac..07dd20e55 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.IO; using System.Reflection; using System.Runtime.InteropServices; @@ -10,9 +11,11 @@ namespace Python.Runtime /// Implements a Python type that provides access to CLR namespaces. The /// type behaves like a Python module, and can contain other sub-modules. /// + [Serializable] internal class ModuleObject : ExtensionType { private Dictionary cache; + internal string moduleName; internal IntPtr dict; protected string _namespace; @@ -45,15 +48,16 @@ public ModuleObject(string name) IntPtr pyfilename = Runtime.PyString_FromString(filename); IntPtr pydocstring = Runtime.PyString_FromString(docstring); IntPtr pycls = TypeManager.GetTypeHandle(GetType()); - Runtime.PyDict_SetItemString(dict, "__name__", pyname); - Runtime.PyDict_SetItemString(dict, "__file__", pyfilename); - Runtime.PyDict_SetItemString(dict, "__doc__", pydocstring); - Runtime.PyDict_SetItemString(dict, "__class__", pycls); + Runtime.PyDict_SetItem(dict, PyIdentifier.__name__, pyname); + Runtime.PyDict_SetItem(dict, PyIdentifier.__file__, pyfilename); + Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, pydocstring); + Runtime.PyDict_SetItem(dict, PyIdentifier.__class__, pycls); Runtime.XDecref(pyname); Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); + Runtime.XIncref(dict); + SetObjectDict(pyHandle, dict); InitializeModuleMembers(); } @@ -99,51 +103,21 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } // Look for a type in the current namespace. Note that this // includes types, delegates, enums, interfaces and structs. // Only public namespace members are exposed to Python. - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; } - // This is a little repetitive, but it ensures that the right - // thing happens with implicit assembly loading at a reasonable - // cost. Ask the AssemblyManager to do implicit loading for each - // of the steps in the qualified name, then try it again. - bool ignore = name.StartsWith("__"); - if (AssemblyManager.LoadImplicit(qname, !ignore)) - { - if (AssemblyManager.IsValidNamespace(qname)) - { - m = new ModuleObject(qname); - StoreAttribute(name, m); - return m; - } - - type = AssemblyManager.LookupType(qname); - if (type != null) - { - if (!type.IsPublic) - { - return null; - } - c = ClassManager.GetClass(type); - StoreAttribute(name, c); - return c; - } - } - // We didn't find the name, so we may need to see if there is a // generic type with this base name. If so, we'll go ahead and // return it. Note that we store the mapping of the unmangled @@ -168,13 +142,22 @@ public ManagedType GetAttribute(string name, bool guess) return null; } + static void ImportWarning(Exception exception) + { + Exceptions.warn(exception.ToString(), Exceptions.ImportWarning); + } + /// /// Stores an attribute in the instance dict for future lookups. /// private void StoreAttribute(string name, ManagedType ob) { - Runtime.PyDict_SetItemString(dict, name, ob.pyHandle); + if (Runtime.PyDict_SetItemString(dict, name, ob.pyHandle) != 0) + { + throw new PythonException(); + } + ob.IncrRefCount(); cache[name] = ob; } @@ -190,10 +173,17 @@ public void LoadNames() foreach (string name in AssemblyManager.GetNames(_namespace)) { cache.TryGetValue(name, out m); - if (m == null) + if (m != null) { - ManagedType attr = GetAttribute(name, true); + continue; } + IntPtr attr = Runtime.PyDict_GetItemString(dict, name); + // If __dict__ has already set a custom property, skip it. + if (attr != IntPtr.Zero) + { + continue; + } + GetAttribute(name, true); } } @@ -224,6 +214,7 @@ internal void InitializeModuleMembers() mi[0] = method; var m = new ModuleFunctionObject(type, name, mi, allow_threads); StoreAttribute(name, m); + m.DecrRefCount(); } } @@ -236,6 +227,7 @@ internal void InitializeModuleMembers() string name = property.Name; var p = new ModulePropertyObject(property); StoreAttribute(name, p); + p.DecrRefCount(); } } type = type.BaseType; @@ -266,14 +258,25 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return op; } - string name = Runtime.GetManagedString(key); + string name = InternString.GetManagedString(key); if (name == "__dict__") { Runtime.XIncref(self.dict); return self.dict; } - ManagedType attr = self.GetAttribute(name, true); + ManagedType attr = null; + + try + { + attr = self.GetAttribute(name, true); + } + catch (Exception e) + { + Exceptions.SetError(e); + return IntPtr.Zero; + } + if (attr == null) { @@ -293,6 +296,76 @@ public static IntPtr tp_repr(IntPtr ob) var self = (ModuleObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + public new static void tp_dealloc(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + tp_clear(ob); + self.Dealloc(); + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ModuleObject)GetManagedObject(ob); + int res = PyVisit(self.dict, visit, arg); + if (res != 0) return res; + foreach (var attr in self.cache.Values) + { + res = PyVisit(attr.pyHandle, visit, arg); + if (res != 0) return res; + } + return 0; + } + + public static int tp_clear(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.dict); + ClearObjectDict(ob); + foreach (var attr in self.cache.Values) + { + Runtime.XDecref(attr.pyHandle); + } + self.cache.Clear(); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + System.Diagnostics.Debug.Assert(dict == GetObjectDict(pyHandle)); + foreach (var attr in cache.Values) + { + Runtime.XIncref(attr.pyHandle); + } + // Decref twice in tp_clear, equilibrate them. + Runtime.XIncref(dict); + Runtime.XIncref(dict); + // destroy the cache(s) + foreach (var pair in cache) + { + if ((Runtime.PyDict_DelItemString(dict, pair.Key) == -1) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } + pair.Value.DecrRefCount(); + } + + cache.Clear(); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + SetObjectDict(pyHandle, dict); + } } /// @@ -300,6 +373,7 @@ public static IntPtr tp_repr(IntPtr ob) /// to import assemblies. It has a fixed module name "clr" and doesn't /// provide a namespace. /// + [Serializable] internal class CLRModule : ModuleObject { protected static bool hacked = false; @@ -309,6 +383,11 @@ internal class CLRModule : ModuleObject internal static bool _SuppressDocs = false; internal static bool _SuppressOverloads = false; + static CLRModule() + { + Reset(); + } + public CLRModule() : base("clr") { _namespace = string.Empty; @@ -328,6 +407,17 @@ public CLRModule() : base("clr") } } + public static void Reset() + { + hacked = false; + interactive_preload = true; + preload = false; + + // XXX Test performance of new features // + _SuppressDocs = false; + _SuppressOverloads = false; + } + /// /// The initializing of the preload hook has to happen as late as /// possible since sys.ps1 is created after the CLR module is @@ -338,7 +428,7 @@ internal void InitializePreload() if (interactive_preload) { interactive_preload = false; - if (Runtime.PySys_GetObject("ps1") != IntPtr.Zero) + if (!Runtime.PySys_GetObject("ps1").IsNull) { preload = true; } @@ -403,6 +493,23 @@ public static Assembly AddReference(string name) return assembly; } + /// + /// Get a Type instance for a class object. + /// clr.GetClrType(IComparable) gives you the Type for IComparable, + /// that you can e.g. perform reflection on. Similar to typeof(IComparable) in C# + /// or clr.GetClrType(IComparable) in IronPython. + /// + /// + /// + /// The Type object + + [ModuleFunction] + [ForbidPythonThreads] + public static Type GetClrType(Type type) + { + return type; + } + [ModuleFunction] [ForbidPythonThreads] public static string FindAssembly(string name) diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs new file mode 100644 index 000000000..e95b259c5 --- /dev/null +++ b/src/runtime/native/ABI.cs @@ -0,0 +1,37 @@ +namespace Python.Runtime.Native +{ + using System; + using System.Globalization; + using System.Linq; + using System.Reflection; + using System.Runtime.InteropServices; + + static class ABI + { + internal static void Initialize(Version version, BorrowedReference pyType) + { + string offsetsClassSuffix = string.Format(CultureInfo.InvariantCulture, + "{0}{1}", version.Major, version.Minor); + + var thisAssembly = Assembly.GetExecutingAssembly(); + + const string nativeTypeOffsetClassName = "Python.Runtime.NativeTypeOffset"; + string className = "Python.Runtime.TypeOffset" + offsetsClassSuffix; + Type typeOffsetsClass = + // Try platform native offsets first. It is only present when generated by setup.py + thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false) + ?? thisAssembly.GetType(className, throwOnError: false); + if (typeOffsetsClass is null) + { + var types = thisAssembly.GetTypes().Select(type => type.Name).Where(name => name.StartsWith("TypeOffset")); + string message = $"Searching for {className}, found {string.Join(",", types)}. " + + "If you are building Python.NET from source, make sure you have run 'python setup.py configure' to fill in configured.props"; + throw new NotSupportedException($"Python ABI v{version} is not supported: {message}"); + } + var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); + TypeOffset.Use(typeOffsets); + + ManagedDataOffsets.Magic = Marshal.ReadInt32(pyType.DangerousGetAddress(), TypeOffset.tp_basicsize); + } + } +} diff --git a/src/runtime/native/GeneratedTypeOffsets.cs b/src/runtime/native/GeneratedTypeOffsets.cs new file mode 100644 index 000000000..bf10df124 --- /dev/null +++ b/src/runtime/native/GeneratedTypeOffsets.cs @@ -0,0 +1,22 @@ +namespace Python.Runtime.Native +{ + using System; + using System.Reflection; + using System.Runtime.InteropServices; + + [StructLayout(LayoutKind.Sequential)] + abstract class GeneratedTypeOffsets + { + protected GeneratedTypeOffsets() + { + var type = this.GetType(); + var offsetProperties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public); + int fieldSize = IntPtr.Size; + for (int fieldIndex = 0; fieldIndex < offsetProperties.Length; fieldIndex++) + { + int offset = fieldIndex * fieldSize; + offsetProperties[fieldIndex].SetValue(this, offset, index: null); + } + } + } +} diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs new file mode 100644 index 000000000..485c041f8 --- /dev/null +++ b/src/runtime/native/ITypeOffsets.cs @@ -0,0 +1,70 @@ +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo +namespace Python.Runtime.Native +{ + using System.Diagnostics.CodeAnalysis; + + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + interface ITypeOffsets + { + int bf_getbuffer { get; } + int mp_ass_subscript { get; } + int mp_length { get; } + int mp_subscript { get; } + int name { get; } + int nb_positive { get; } + int nb_negative { get; } + int nb_add { get; } + int nb_subtract { get; } + int nb_multiply { get; } + int nb_true_divide { get; } + int nb_and { get; } + int nb_or { get; } + int nb_xor { get; } + int nb_lshift { get; } + int nb_rshift { get; } + int nb_remainder { get; } + int nb_invert { get; } + int nb_inplace_add { get; } + int nb_inplace_subtract { get; } + int ob_size { get; } + int ob_type { get; } + int qualname { get; } + int sq_contains { get; } + int sq_length { get; } + int tp_alloc { get; } + int tp_as_buffer { get; } + int tp_as_mapping { get; } + int tp_as_number { get; } + int tp_as_sequence { get; } + int tp_base { get; } + int tp_bases { get; } + int tp_basicsize { get; } + int tp_call { get; } + int tp_clear { get; } + int tp_dealloc { get; } + int tp_descr_get { get; } + int tp_descr_set { get; } + int tp_dict { get; } + int tp_dictoffset { get; } + int tp_flags { get; } + int tp_free { get; } + int tp_getattro { get; } + int tp_hash { get; } + int tp_is_gc { get; } + int tp_itemsize { get; } + int tp_iter { get; } + int tp_iternext { get; } + int tp_methods { get; } + int tp_mro { get; } + int tp_name { get; } + int tp_new { get; } + int tp_repr { get; } + int tp_richcompare { get; } + int tp_setattro { get; } + int tp_str { get; } + int tp_traverse { get; } + } +} diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs new file mode 100644 index 000000000..4c1bcefa0 --- /dev/null +++ b/src/runtime/native/TypeOffset.cs @@ -0,0 +1,172 @@ +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo +namespace Python.Runtime +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.Linq; + using System.Reflection; + + using Python.Runtime.Native; + + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + static partial class TypeOffset + { + internal static int bf_getbuffer { get; private set; } + internal static int mp_ass_subscript { get; private set; } + internal static int mp_length { get; private set; } + internal static int mp_subscript { get; private set; } + internal static int name { get; private set; } + internal static int nb_positive { get; private set; } + internal static int nb_negative { get; private set; } + internal static int nb_add { get; private set; } + internal static int nb_subtract { get; private set; } + internal static int nb_multiply { get; private set; } + internal static int nb_true_divide { get; private set; } + internal static int nb_and { get; private set; } + internal static int nb_or { get; private set; } + internal static int nb_xor { get; private set; } + internal static int nb_lshift { get; private set; } + internal static int nb_rshift { get; private set; } + internal static int nb_remainder { get; private set; } + internal static int nb_invert { get; private set; } + internal static int nb_inplace_add { get; private set; } + internal static int nb_inplace_subtract { get; private set; } + internal static int ob_size { get; private set; } + internal static int ob_type { get; private set; } + internal static int qualname { get; private set; } + internal static int sq_contains { get; private set; } + internal static int sq_length { get; private set; } + internal static int tp_alloc { get; private set; } + internal static int tp_as_buffer { get; private set; } + internal static int tp_as_mapping { get; private set; } + internal static int tp_as_number { get; private set; } + internal static int tp_as_sequence { get; private set; } + internal static int tp_base { get; private set; } + internal static int tp_bases { get; private set; } + internal static int tp_basicsize { get; private set; } + internal static int tp_call { get; private set; } + internal static int tp_clear { get; private set; } + internal static int tp_dealloc { get; private set; } + internal static int tp_descr_get { get; private set; } + internal static int tp_descr_set { get; private set; } + internal static int tp_dict { get; private set; } + internal static int tp_dictoffset { get; private set; } + internal static int tp_flags { get; private set; } + internal static int tp_free { get; private set; } + internal static int tp_getattro { get; private set; } + internal static int tp_hash { get; private set; } + internal static int tp_is_gc { get; private set; } + internal static int tp_itemsize { get; private set; } + internal static int tp_iter { get; private set; } + internal static int tp_iternext { get; private set; } + internal static int tp_methods { get; private set; } + internal static int tp_mro { get; private set; } + internal static int tp_name { get; private set; } + internal static int tp_new { get; private set; } + internal static int tp_repr { get; private set; } + internal static int tp_richcompare { get; private set; } + internal static int tp_setattro { get; private set; } + internal static int tp_str { get; private set; } + internal static int tp_traverse { get; private set; } + + internal static void Use(ITypeOffsets offsets) + { + if (offsets is null) throw new ArgumentNullException(nameof(offsets)); + + slotNames.Clear(); + var offsetProperties = typeof(TypeOffset).GetProperties(FieldFlags); + foreach (var offsetProperty in offsetProperties) + { + slotNames.Add(offsetProperty.Name); + + var sourceProperty = typeof(ITypeOffsets).GetProperty(offsetProperty.Name); + int value = (int)sourceProperty.GetValue(offsets, null); + offsetProperty.SetValue(obj: null, value: value, index: null); + } + + ValidateUnusedTypeOffsetProperties(offsetProperties); + ValidateRequiredOffsetsPresent(offsetProperties); + } + + static readonly BindingFlags FieldFlags = BindingFlags.NonPublic | BindingFlags.Static; + internal static Dictionary GetOffsets() + { + var properties = typeof(TypeOffset).GetProperties(FieldFlags); + var result = properties.ToDictionary( + keySelector: p => p.Name, + elementSelector: p => (int)p.GetValue(obj: null, index: null)); + Debug.Assert(result.Values.Any(v => v != 0)); + return result; + } + + internal static int GetOffsetUncached(string name) + { + var property = typeof(TypeOffset).GetProperty(name, FieldFlags); + return (int)property.GetValue(obj: null, index: null); + } + + static readonly HashSet slotNames = new HashSet(); + internal static bool IsSupportedSlotName(string name) => slotNames.Contains(name); + + [Conditional("DEBUG")] + static void ValidateUnusedTypeOffsetProperties(PropertyInfo[] offsetProperties) + { + var extras = new List(); + foreach (var property in typeof(ITypeOffsets).GetProperties(FieldFlags)) + { + if (!offsetProperties.Any(prop => prop.Name == property.Name)) + extras.Add(property.Name); + } + extras.Sort(); + Debug.Assert(extras.Count == 0, message: string.Join(", ", extras)); + } + + [Conditional("DEBUG")] + static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) + { + var present = new HashSet(offsetProperties.Select(p => p.Name)); + var missing = new HashSet(); + + var thisAssembly = Assembly.GetExecutingAssembly(); + var managedTypes = thisAssembly.GetTypes() + .Where(typeof(ManagedType).IsAssignableFrom) + .ToList(); + foreach (var managedType in managedTypes) + { + var slots = managedType.GetMethods(BindingFlags.Public | BindingFlags.Static); + foreach(var slot in slots) + if (!present.Contains(slot.Name)) + missing.Add(slot.Name); + } + foreach (string notSlot in new[] + { + "__instancecheck__", + "__subclasscheck__", + "_AtExit", + "AddReference", + "FinalizeObject", + "FindAssembly", + "get_SuppressDocs", + "get_SuppressOverloads", + "GetClrType", + "getPreload", + "Initialize", + "ListAssemblies", + "Release", + "Reset", + "set_SuppressDocs", + "set_SuppressOverloads", + "setPreload", + }) + missing.Remove(notSlot); + + Debug.Assert(missing.Count == 0, + "Missing slots: " + string.Join(", ", missing)); + } + } +} diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index 9d1b0f41c..60259dcd0 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -12,140 +12,43 @@ namespace Python.Runtime /// C API can just be wrapped with p/invoke, but there are some /// situations (specifically, calling functions through Python /// type structures) where we need to call functions indirectly. - /// This class uses Reflection.Emit to generate IJW thunks that - /// support indirect calls to native code using various common - /// call signatures. This is mainly a workaround for the fact - /// that you can't spell an indirect call in C# (but can in IL). - /// Another approach that would work is for this to be turned - /// into a separate utility program that could be run during the - /// build process to generate the thunks as a separate assembly - /// that could then be referenced by the main Python runtime. /// internal class NativeCall { - private static AssemblyBuilder aBuilder; - private static ModuleBuilder mBuilder; - - public static INativeCall Impl; - - static NativeCall() - { - // The static constructor is responsible for generating the - // assembly and the methods that implement the IJW thunks. - // - // To do this, we actually use reflection on the INativeCall - // interface (defined below) and generate the required thunk - // code based on the method signatures. - - var aname = new AssemblyName { Name = "e__NativeCall_Assembly" }; - var aa = AssemblyBuilderAccess.Run; - - aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("e__NativeCall_Module"); - - var ta = TypeAttributes.Public; - TypeBuilder tBuilder = mBuilder.DefineType("e__NativeCall", ta); - - Type iType = typeof(INativeCall); - tBuilder.AddInterfaceImplementation(iType); - - // Use reflection to loop over the INativeCall interface methods, - // calling GenerateThunk to create a managed thunk for each one. - - foreach (MethodInfo method in iType.GetMethods()) - { - GenerateThunk(tBuilder, method); - } - - Type theType = tBuilder.CreateType(); - - Impl = (INativeCall)Activator.CreateInstance(theType); - } - - private static void GenerateThunk(TypeBuilder tb, MethodInfo method) - { - ParameterInfo[] pi = method.GetParameters(); - int count = pi.Length; - int argc = count - 1; - - var args = new Type[count]; - for (var i = 0; i < count; i++) - { - args[i] = pi[i].ParameterType; - } - - MethodBuilder mb = tb.DefineMethod( - method.Name, - MethodAttributes.Public | - MethodAttributes.Virtual, - method.ReturnType, - args - ); - - // Build the method signature for the actual native function. - // This is essentially the signature of the wrapper method - // minus the first argument (the passed in function pointer). - - var nargs = new Type[argc]; - for (var i = 1; i < count; i++) - { - nargs[i - 1] = args[i]; - } - - // IL generation: the (implicit) first argument of the method - // is the 'this' pointer and the second is the function pointer. - // This code pushes the real args onto the stack, followed by - // the function pointer, then the calli opcode to make the call. - - ILGenerator il = mb.GetILGenerator(); - - for (var i = 0; i < argc; i++) - { - il.Emit(OpCodes.Ldarg_S, i + 2); - } - - il.Emit(OpCodes.Ldarg_1); - - il.EmitCalli(OpCodes.Calli, - CallingConvention.Cdecl, - method.ReturnType, - nargs - ); - - il.Emit(OpCodes.Ret); - - tb.DefineMethodOverride(mb, method); - } + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate void Void_1_Delegate(IntPtr a1); + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int Int_3_Delegate(IntPtr a1, IntPtr a2, IntPtr a3); public static void Void_Call_1(IntPtr fp, IntPtr a1) { - Impl.Void_Call_1(fp, a1); + var d = GetDelegate(fp); + d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - return Impl.Call_3(fp, a1, a2, a3); + var d = GetDelegate(fp); + return d(a1, a2, a3); } + public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - return Impl.Int_Call_3(fp, a1, a2, a3); + var d = GetDelegate(fp); + return d(a1, a2, a3); } - } - - /// - /// Defines native call signatures to be generated by NativeCall. - /// - public interface INativeCall - { - void Void_Call_0(IntPtr funcPtr); - - void Void_Call_1(IntPtr funcPtr, IntPtr arg1); - - int Int_Call_3(IntPtr funcPtr, IntPtr t, IntPtr n, IntPtr v); - - IntPtr Call_3(IntPtr funcPtr, IntPtr a1, IntPtr a2, IntPtr a3); + internal static T GetDelegate(IntPtr fp) where T: Delegate + { + Delegate d = null; + if (!Interop.allocatedThunks.TryGetValue(fp, out d)) + { + // We don't cache this delegate because this is a pure delegate ot unmanaged. + d = Marshal.GetDelegateForFunctionPointer(fp); + } + return (T)d; + } } } diff --git a/src/runtime/operatormethod.cs b/src/runtime/operatormethod.cs new file mode 100644 index 000000000..59bf944bc --- /dev/null +++ b/src/runtime/operatormethod.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime +{ + internal static class OperatorMethod + { + /// + /// Maps the compiled method name in .NET CIL (e.g. op_Addition) to + /// the equivalent Python operator (e.g. __add__) as well as the offset + /// that identifies that operator's slot (e.g. nb_add) in heap space. + /// + public static Dictionary OpMethodMap { get; private set; } + public static Dictionary ComparisonOpMap { get; private set; } + public readonly struct SlotDefinition + { + public SlotDefinition(string methodName, int typeOffset) + { + MethodName = methodName; + TypeOffset = typeOffset; + } + public string MethodName { get; } + public int TypeOffset { get; } + + } + private static PyObject _opType; + + static OperatorMethod() + { + // .NET operator method names are documented at: + // https://docs.microsoft.com/en-us/dotnet/standard/design-guidelines/operator-overloads + // Python operator methods and slots are documented at: + // https://docs.python.org/3/c-api/typeobj.html + // TODO: Rich compare, inplace operator support + OpMethodMap = new Dictionary + { + ["op_Addition"] = new SlotDefinition("__add__", TypeOffset.nb_add), + ["op_Subtraction"] = new SlotDefinition("__sub__", TypeOffset.nb_subtract), + ["op_Multiply"] = new SlotDefinition("__mul__", TypeOffset.nb_multiply), + ["op_Division"] = new SlotDefinition("__truediv__", TypeOffset.nb_true_divide), + ["op_Modulus"] = new SlotDefinition("__mod__", TypeOffset.nb_remainder), + ["op_BitwiseAnd"] = new SlotDefinition("__and__", TypeOffset.nb_and), + ["op_BitwiseOr"] = new SlotDefinition("__or__", TypeOffset.nb_or), + ["op_ExclusiveOr"] = new SlotDefinition("__xor__", TypeOffset.nb_xor), + ["op_LeftShift"] = new SlotDefinition("__lshift__", TypeOffset.nb_lshift), + ["op_RightShift"] = new SlotDefinition("__rshift__", TypeOffset.nb_rshift), + ["op_OnesComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), + ["op_UnaryNegation"] = new SlotDefinition("__neg__", TypeOffset.nb_negative), + ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), + ["op_OneComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), + }; + ComparisonOpMap = new Dictionary + { + ["op_Equality"] = "__eq__", + ["op_Inequality"] = "__ne__", + ["op_LessThanOrEqual"] = "__le__", + ["op_GreaterThanOrEqual"] = "__ge__", + ["op_LessThan"] = "__lt__", + ["op_GreaterThan"] = "__gt__", + }; + } + + public static void Initialize() + { + _opType = GetOperatorType(); + } + + public static void Shutdown() + { + if (_opType != null) + { + _opType.Dispose(); + _opType = null; + } + } + + public static bool IsOperatorMethod(MethodBase method) + { + if (!method.IsSpecialName) + { + return false; + } + return OpMethodMap.ContainsKey(method.Name) || ComparisonOpMap.ContainsKey(method.Name); + } + + public static bool IsComparisonOp(MethodInfo method) + { + return ComparisonOpMap.ContainsKey(method.Name); + } + + /// + /// For the operator methods of a CLR type, set the special slots of the + /// corresponding Python type's operator methods. + /// + /// + /// + public static void FixupSlots(IntPtr pyType, Type clrType) + { + const BindingFlags flags = BindingFlags.Public | BindingFlags.Static; + Debug.Assert(_opType != null); + foreach (var method in clrType.GetMethods(flags)) + { + // We only want to override slots for operators excluding + // comparison operators, which are handled by ClassBase.tp_richcompare. + if (!OpMethodMap.ContainsKey(method.Name)) + { + continue; + } + int offset = OpMethodMap[method.Name].TypeOffset; + // Copy the default implementation of e.g. the nb_add slot, + // which simply calls __add__ on the type. + IntPtr func = Marshal.ReadIntPtr(_opType.Handle, offset); + // Write the slot definition of the target Python type, so + // that we can later modify __add___ and it will be called + // when used with a Python operator. + // https://tenthousandmeters.com/blog/python-behind-the-scenes-6-how-python-object-system-works/ + Marshal.WriteIntPtr(pyType, offset, func); + } + } + + public static string GetPyMethodName(string clrName) + { + if (OpMethodMap.ContainsKey(clrName)) + { + return OpMethodMap[clrName].MethodName; + } else + { + return ComparisonOpMap[clrName]; + } + } + + private static string GenerateDummyCode() + { + StringBuilder sb = new StringBuilder(); + sb.AppendLine("class OperatorMethod(object):"); + foreach (var item in OpMethodMap.Values) + { + string def = string.Format(" def {0}(self, other): pass", item.MethodName); + sb.AppendLine(def); + } + return sb.ToString(); + } + + private static PyObject GetOperatorType() + { + using (PyDict locals = new PyDict()) + { + // A hack way for getting typeobject.c::slotdefs + string code = GenerateDummyCode(); + // The resulting OperatorMethod class is stored in a PyDict. + PythonEngine.Exec(code, null, locals.Handle); + // Return the class itself, which is a type. + return locals.GetItem("OperatorMethod"); + } + } + + public static string ReversePyMethodName(string pyName) + { + return pyName.Insert(2, "r"); + } + + /// + /// Check if the method is performing a reverse operation. + /// + /// The operator method. + /// + public static bool IsReverse(MethodInfo method) + { + Type declaringType = method.DeclaringType; + Type leftOperandType = method.GetParameters()[0].ParameterType; + return leftOperandType != declaringType; + } + + public static void FilterMethods(MethodInfo[] methods, out MethodInfo[] forwardMethods, out MethodInfo[] reverseMethods) + { + List forwardMethodsList = new List(); + List reverseMethodsList = new List(); + foreach (var method in methods) + { + if (IsReverse(method)) + { + reverseMethodsList.Add(method); + } else + { + forwardMethodsList.Add(method); + } + + } + forwardMethods = forwardMethodsList.ToArray(); + reverseMethods = reverseMethodsList.ToArray(); + } + } +} diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index 6b48299e8..e9fa91d3b 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -44,7 +44,6 @@ public static IntPtr mp_subscript(IntPtr tp, IntPtr idx) } var mb = new MethodBinding(self.m, self.target) { info = mi }; - Runtime.XIncref(mb.pyHandle); return mb.pyHandle; } @@ -66,7 +65,7 @@ public static IntPtr tp_repr(IntPtr op) { var self = (OverloadMapper)GetManagedObject(ob); Runtime.XDecref(self.target); - FinalizeObject(self); + self.Dealloc(); } } } diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs new file mode 100644 index 000000000..193dff274 --- /dev/null +++ b/src/runtime/platform/LibraryLoader.cs @@ -0,0 +1,217 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibraryLoader + { + IntPtr Load(string dllToLoad); + + IntPtr GetFunction(IntPtr hModule, string procedureName); + + void Free(IntPtr hModule); + } + + static class LibraryLoader + { + static ILibraryLoader _instance = null; + + public static ILibraryLoader Instance + { + get + { + if (_instance == null) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + _instance = new WindowsLoader(); + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + _instance = new LinuxLoader(); + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + _instance = new DarwinLoader(); + else + throw new PlatformNotSupportedException( + "This operating system is not supported" + ); + } + + return _instance; + } + } + } + + class LinuxLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x100; + private static IntPtr RTLD_DEFAULT = IntPtr.Zero; + private const string NativeDll = "libdl.so"; + + public IntPtr Load(string dllToLoad) + { + var filename = $"lib{dllToLoad}.so"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; + } + + public void Free(IntPtr handle) + { + dlclose(handle); + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + ClearError(); + IntPtr res = dlsym(dllHandle, name); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); + } + return res; + } + + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class DarwinLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x8; + private const string NativeDll = "/usr/lib/libSystem.dylib"; + private static IntPtr RTLD_DEFAULT = new IntPtr(-2); + + public IntPtr Load(string dllToLoad) + { + var filename = $"lib{dllToLoad}.dylib"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; + } + + public void Free(IntPtr handle) + { + dlclose(handle); + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + ClearError(); + IntPtr res = dlsym(dllHandle, name); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); + } + return res; + } + + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class WindowsLoader : ILibraryLoader + { + private const string NativeDll = "kernel32.dll"; + + + public IntPtr Load(string dllToLoad) + { + var res = WindowsLoader.LoadLibrary(dllToLoad); + if (res == IntPtr.Zero) + throw new DllNotFoundException($"Could not load {dllToLoad}", new Win32Exception()); + return res; + } + + public IntPtr GetFunction(IntPtr hModule, string procedureName) + { + var res = WindowsLoader.GetProcAddress(hModule, procedureName); + if (res == IntPtr.Zero) + throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); + return res; + } + + public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr LoadLibrary(string dllToLoad); + + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + + [DllImport(NativeDll)] + static extern bool FreeLibrary(IntPtr hModule); + } +} diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs new file mode 100644 index 000000000..b6fe89425 --- /dev/null +++ b/src/runtime/platform/NativeCodePage.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + class NativeCodePageHelper + { + /// + /// Initialized by InitializeNativeCodePage. + /// + /// This points to a page of memory allocated using mmap or VirtualAlloc + /// (depending on the system), and marked read and execute (not write). + /// Very much on purpose, the page is *not* released on a shutdown and + /// is instead leaked. See the TestDomainReload test case. + /// + /// The contents of the page are two native functions: one that returns 0, + /// one that returns 1. + /// + /// If python didn't keep its gc list through a Py_Finalize we could remove + /// this entire section. + /// + internal static IntPtr NativeCodePage = IntPtr.Zero; + + + /// + /// Structure to describe native code. + /// + /// Use NativeCode.Active to get the native code for the current platform. + /// + /// Generate the code by creating the following C code: + /// + /// int Return0() { return 0; } + /// int Return1() { return 1; } + /// + /// Then compiling on the target platform, e.g. with gcc or clang: + /// cc -c -fomit-frame-pointer -O2 foo.c + /// And then analyzing the resulting functions with a hex editor, e.g.: + /// objdump -disassemble foo.o + /// + internal class NativeCode + { + /// + /// The code, as a string of bytes. + /// + public byte[] Code { get; private set; } + + /// + /// Where does the "return 0" function start? + /// + public int Return0 { get; private set; } + + /// + /// Where does the "return 1" function start? + /// + public int Return1 { get; private set; } + + public static NativeCode Active + { + get + { + switch (RuntimeInformation.ProcessArchitecture) + { + case Architecture.X86: + return I386; + case Architecture.X64: + return X86_64; + default: + return null; + } + } + } + + /// + /// Code for x86_64. See the class comment for how it was generated. + /// + public static readonly NativeCode X86_64 = new NativeCode() + { + Return0 = 0x10, + Return1 = 0, + Code = new byte[] + { + // First Return1: + 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax + 0xc3, // ret + + // Now some padding so that Return0 can be 16-byte-aligned. + // I put Return1 first so there's not as much padding to type in. + 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop + + // Now Return0. + 0x31, 0xc0, // xorl %eax, %eax + 0xc3, // ret + } + }; + + /// + /// Code for X86. + /// + /// It's bitwise identical to X86_64, so we just point to it. + /// + /// + public static readonly NativeCode I386 = X86_64; + } + + /// + /// Platform-dependent mmap and mprotect. + /// + internal interface IMemoryMapper + { + /// + /// Map at least numBytes of memory. Mark the page read-write (but not exec). + /// + IntPtr MapWriteable(int numBytes); + + /// + /// Sets the mapped memory to be read-exec (but not write). + /// + void SetReadExec(IntPtr mappedMemory, int numBytes); + } + + class WindowsMemoryMapper : IMemoryMapper + { + const UInt32 MEM_COMMIT = 0x1000; + const UInt32 MEM_RESERVE = 0x2000; + const UInt32 PAGE_READWRITE = 0x04; + const UInt32 PAGE_EXECUTE_READ = 0x20; + + [DllImport("kernel32.dll")] + static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); + + [DllImport("kernel32.dll")] + static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); + + public IntPtr MapWriteable(int numBytes) + { + return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + UInt32 _; + VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); + } + } + + class UnixMemoryMapper : IMemoryMapper + { + const int PROT_READ = 0x1; + const int PROT_WRITE = 0x2; + const int PROT_EXEC = 0x4; + + const int MAP_PRIVATE = 0x2; + int MAP_ANONYMOUS + { + get + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return 0x20; + } + else + { + // OSX, FreeBSD + return 0x1000; + } + } + } + + [DllImport("libc")] + static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); + + [DllImport("libc")] + static extern int mprotect(IntPtr addr, IntPtr len, int prot); + + public IntPtr MapWriteable(int numBytes) + { + // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. + // It doesn't hurt on darwin, so just do it. + return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); + } + } + + internal static IMemoryMapper CreateMemoryMapper() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return new WindowsMemoryMapper(); + } + else + { + // Linux, OSX, FreeBSD + return new UnixMemoryMapper(); + } + } + + /// + /// Initializes the native code page. + /// + /// Safe to call if we already initialized (this function is idempotent). + /// + /// + internal static void InitializeNativeCodePage() + { + // Do nothing if we already initialized. + if (NativeCodePage != IntPtr.Zero) + { + return; + } + + // Allocate the page, write the native code into it, then set it + // to be executable. + IMemoryMapper mapper = CreateMemoryMapper(); + int codeLength = NativeCode.Active.Code.Length; + NativeCodePage = mapper.MapWriteable(codeLength); + Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); + mapper.SetReadExec(NativeCodePage, codeLength); + } + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs new file mode 100644 index 000000000..15235da5a --- /dev/null +++ b/src/runtime/platform/Types.cs @@ -0,0 +1,192 @@ +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + public enum MachineType + { + i386, + x86_64, + armv7l, + armv8, + aarch64, + Other + }; + + /// + /// Operating system type as reported by Python. + /// + public enum OperatingSystemType + { + Windows, + Darwin, + Linux, + Other + } + + + static class SystemInfo + { + public static MachineType GetMachineType() + { + return Runtime.IsWindows ? GetMachineType_Windows() : GetMachineType_Unix(); + } + + public static string GetArchitecture() + { + return Runtime.IsWindows ? GetArchName_Windows() : GetArchName_Unix(); + } + + public static OperatingSystemType GetSystemType() + { + if (Runtime.IsWindows) + { + return OperatingSystemType.Windows; + } + switch (PythonEngine.Platform) + { + case "linux": + return OperatingSystemType.Linux; + + case "darwin": + return OperatingSystemType.Darwin; + + default: + return OperatingSystemType.Other; + } + } + + #region WINDOWS + + static string GetArchName_Windows() + { + // https://docs.microsoft.com/en-us/windows/win32/winprog64/wow64-implementation-details + return Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE"); + } + + static MachineType GetMachineType_Windows() + { + if (Runtime.Is32Bit) + { + return MachineType.i386; + } + switch (GetArchName_Windows()) + { + case "AMD64": + return MachineType.x86_64; + case "ARM64": + return MachineType.aarch64; + default: + return MachineType.Other; + } + } + + #endregion + + #region UNIX + + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_linux + { + const int NameLength = 65; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + + // GNU extension + fixed byte domainname[NameLength]; /* NIS or YP domain name */ + } + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_darwin + { + const int NameLength = 256; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + } + + [DllImport("libc")] + static extern int uname(IntPtr buf); + + + static unsafe string GetArchName_Unix() + { + switch (GetSystemType()) + { + case OperatingSystemType.Linux: + { + var buf = stackalloc utsname_linux[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + case OperatingSystemType.Darwin: + { + var buf = stackalloc utsname_darwin[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + default: + return null; + } + } + + static unsafe MachineType GetMachineType_Unix() + { + switch (GetArchName_Unix()) + { + case "x86_64": + case "em64t": + return Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; + case "i386": + case "i686": + return MachineType.i386; + + case "armv7l": + return MachineType.armv7l; + case "armv8": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.armv8; + case "aarch64": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.aarch64; + + default: + return MachineType.Other; + } + } + + #endregion + } +} diff --git a/src/runtime/polyfill/ReflectionPolyfills.cs b/src/runtime/polyfill/ReflectionPolyfills.cs new file mode 100644 index 000000000..65f9b83de --- /dev/null +++ b/src/runtime/polyfill/ReflectionPolyfills.cs @@ -0,0 +1,34 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; + +namespace Python.Runtime +{ + internal static class ReflectionPolyfills + { + public static AssemblyBuilder DefineDynamicAssembly(this AppDomain appDomain, AssemblyName assemblyName, AssemblyBuilderAccess assemblyBuilderAccess) + { + return AssemblyBuilder.DefineDynamicAssembly(assemblyName, assemblyBuilderAccess); + } + + public static Type CreateType(this TypeBuilder typeBuilder) + { + return typeBuilder.CreateTypeInfo(); + } + + public static T GetCustomAttribute(this Type type) where T: Attribute + { + return type.GetCustomAttributes(typeof(T), inherit: false) + .Cast() + .SingleOrDefault(); + } + + public static T GetCustomAttribute(this Assembly assembly) where T: Attribute + { + return assembly.GetCustomAttributes(typeof(T), inherit: false) + .Cast() + .SingleOrDefault(); + } + } +} diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..20061b358 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -4,14 +4,16 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python descriptor type that manages CLR properties. /// + [Serializable] internal class PropertyObject : ExtensionType { - private PropertyInfo info; - private MethodInfo getter; - private MethodInfo setter; + private MaybeMemberInfo info; + private MaybeMethodInfo getter; + private MaybeMethodInfo setter; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -30,7 +32,12 @@ public PropertyObject(PropertyInfo md) public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo getter = self.getter; + if (!self.info.Valid) + { + return Exceptions.RaiseTypeError(self.info.DeletedMessage); + } + var info = self.info.Value; + MethodInfo getter = self.getter.UnsafeValue; object result; @@ -50,8 +57,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(null, null); - return Converter.ToPython(result, self.info.PropertyType); + result = info.GetValue(null, null); + return Converter.ToPython(result, info.PropertyType); } catch (Exception e) { @@ -67,8 +74,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); - return Converter.ToPython(result, self.info.PropertyType); + result = info.GetValue(co.inst, null); + return Converter.ToPython(result, info.PropertyType); } catch (Exception e) { @@ -90,7 +97,14 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public new static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo setter = self.setter; + if (!self.info.Valid) + { + Exceptions.RaiseTypeError(self.info.DeletedMessage); + return -1; + } + var info = self.info.Value; + + MethodInfo setter = self.setter.UnsafeValue; object newval; if (val == IntPtr.Zero) @@ -106,7 +120,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } - if (!Converter.ToManaged(val, self.info.PropertyType, out newval, true)) + if (!Converter.ToManaged(val, info.PropertyType, out newval, true)) { return -1; } @@ -132,11 +146,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + info.SetValue(co.inst, newval, null); } else { - self.info.SetValue(null, newval, null); + info.SetValue(null, newval, null); } return 0; } @@ -158,7 +172,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (PropertyObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/pyansistring.cs b/src/runtime/pyansistring.cs index 3d1d6ab68..8a27104b1 100644 --- a/src/runtime/pyansistring.cs +++ b/src/runtime/pyansistring.cs @@ -17,6 +17,16 @@ public PyAnsiString(IntPtr ptr) : base(ptr) } + private static IntPtr FromObject(PyObject o) + { + if (o == null || !IsStringType(o)) + { + throw new ArgumentException("object is not a string"); + } + Runtime.XIncref(o.obj); + return o.obj; + } + /// /// PyString Constructor /// @@ -25,14 +35,14 @@ public PyAnsiString(IntPtr ptr) : base(ptr) /// An ArgumentException will be thrown if the given object is not /// a Python string object. /// - public PyAnsiString(PyObject o) + public PyAnsiString(PyObject o) : base(FromObject(o)) { - if (!IsStringType(o)) - { - throw new ArgumentException("object is not a string"); - } - Runtime.XIncref(o.obj); - obj = o.obj; + } + private static IntPtr FromString(string s) + { + IntPtr val = Runtime.PyString_FromString(s); + PythonException.ThrowIfIsNull(val); + return val; } @@ -42,10 +52,8 @@ public PyAnsiString(PyObject o) /// /// Creates a Python string from a managed string. /// - public PyAnsiString(string s) + public PyAnsiString(string s) : base(FromString(s)) { - obj = Runtime.PyString_FromString(s); - Runtime.CheckExceptionOccurred(); } diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs new file mode 100644 index 000000000..cf657a033 --- /dev/null +++ b/src/runtime/pybuffer.cs @@ -0,0 +1,252 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime +{ + public sealed class PyBuffer : IDisposable + { + private PyObject _exporter; + private Py_buffer _view; + + unsafe internal PyBuffer(PyObject exporter, PyBUF flags) + { + _view = new Py_buffer(); + + if (Runtime.PyObject_GetBuffer(exporter.Handle, ref _view, (int)flags) < 0) + { + throw new PythonException(); + } + + _exporter = exporter; + + var intPtrBuf = new IntPtr[_view.ndim]; + if (_view.shape != IntPtr.Zero) + { + Marshal.Copy(_view.shape, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr)); + Shape = intPtrBuf.Select(x => (long)x).ToArray(); + } + + if (_view.strides != IntPtr.Zero) { + Marshal.Copy(_view.strides, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr)); + Strides = intPtrBuf.Select(x => (long)x).ToArray(); + } + + if (_view.suboffsets != IntPtr.Zero) { + Marshal.Copy(_view.suboffsets, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr)); + SubOffsets = intPtrBuf.Select(x => (long)x).ToArray(); + } + } + + public PyObject Object => _exporter; + public long Length => (long)_view.len; + public long ItemSize => (long)_view.itemsize; + public int Dimensions => _view.ndim; + public bool ReadOnly => _view._readonly; + public IntPtr Buffer => _view.buf; + public string Format => _view.format; + + /// + /// An array of length indicating the shape of the memory as an n-dimensional array. + /// + public long[] Shape { get; private set; } + + /// + /// An array of length giving the number of bytes to skip to get to a new element in each dimension. + /// Will be null except when PyBUF_STRIDES or PyBUF_INDIRECT flags in GetBuffer/>. + /// + public long[] Strides { get; private set; } + + /// + /// An array of Py_ssize_t of length ndim. If suboffsets[n] >= 0, + /// the values stored along the nth dimension are pointers and the suboffset value dictates how many bytes to add to each pointer after de-referencing. + /// A suboffset value that is negative indicates that no de-referencing should occur (striding in a contiguous memory block). + /// + public long[] SubOffsets { get; private set; } + + private static char OrderStyleToChar(BufferOrderStyle order, bool eitherOneValid) + { + char style = 'C'; + if (order == BufferOrderStyle.C) + style = 'C'; + else if (order == BufferOrderStyle.Fortran) + style = 'F'; + else if (order == BufferOrderStyle.EitherOne) + { + if (eitherOneValid) style = 'A'; + else throw new ArgumentException("BufferOrderStyle can not be EitherOne and has to be C or Fortran"); + } + return style; + } + + /// + /// Return the implied itemsize from format. On error, raise an exception and return -1. + /// New in version 3.9. + /// + public static long SizeFromFormat(string format) + { + if (Runtime.PyVersion < new Version(3,9)) + throw new NotSupportedException("SizeFromFormat requires at least Python 3.9"); + return (long)Runtime.PyBuffer_SizeFromFormat(format); + } + + /// + /// Returns true if the memory defined by the view is C-style (order is 'C') or Fortran-style (order is 'F') contiguous or either one (order is 'A'). Returns false otherwise. + /// + /// C-style (order is 'C') or Fortran-style (order is 'F') contiguous or either one (order is 'A') + public bool IsContiguous(BufferOrderStyle order) + { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + return Convert.ToBoolean(Runtime.PyBuffer_IsContiguous(ref _view, OrderStyleToChar(order, true))); + } + + /// + /// Get the memory area pointed to by the indices inside the given view. indices must point to an array of view->ndim indices. + /// + public IntPtr GetPointer(long[] indices) + { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + if (Runtime.PyVersion < new Version(3, 7)) + throw new NotSupportedException("GetPointer requires at least Python 3.7"); + return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => (IntPtr)x).ToArray()); + } + + /// + /// Copy contiguous len bytes from buf to view. fort can be 'C' or 'F' (for C-style or Fortran-style ordering). + /// + public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort) + { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + if (Runtime.PyVersion < new Version(3, 7)) + throw new NotSupportedException("FromContiguous requires at least Python 3.7"); + + if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0) + throw new PythonException(); + } + + /// + /// Copy len bytes from view to its contiguous representation in buf. order can be 'C' or 'F' or 'A' (for C-style or Fortran-style ordering or either one). 0 is returned on success, -1 on error. + /// + /// order can be 'C' or 'F' or 'A' (for C-style or Fortran-style ordering or either one). + /// Buffer to copy to + public void ToContiguous(IntPtr buf, BufferOrderStyle order) + { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + + if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0) + throw new PythonException(); + } + + /// + /// Fill the strides array with byte-strides of a contiguous (C-style if order is 'C' or Fortran-style if order is 'F') array of the given shape with the given number of bytes per element. + /// + public static void FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, BufferOrderStyle order) + { + Runtime.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, OrderStyleToChar(order, false)); + } + + /// + /// FillInfo Method + /// + /// + /// Handle buffer requests for an exporter that wants to expose buf of size len with writability set according to readonly. buf is interpreted as a sequence of unsigned bytes. + /// The flags argument indicates the request type. This function always fills in view as specified by flags, unless buf has been designated as read-only and PyBUF_WRITABLE is set in flags. + /// On success, set view->obj to a new reference to exporter and return 0. Otherwise, raise PyExc_BufferError, set view->obj to NULL and return -1; + /// If this function is used as part of a getbufferproc, exporter MUST be set to the exporting object and flags must be passed unmodified.Otherwise, exporter MUST be NULL. + /// + /// On success, set view->obj to a new reference to exporter and return 0. Otherwise, raise PyExc_BufferError, set view->obj to NULL and return -1; + public void FillInfo(IntPtr exporter, IntPtr buf, long len, bool _readonly, int flags) + { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + if (Runtime.PyBuffer_FillInfo(ref _view, exporter, buf, (IntPtr)len, Convert.ToInt32(_readonly), flags) < 0) + throw new PythonException(); + } + + /// + /// Writes a managed byte array into the buffer of a python object. This can be used to pass data like images from managed to python. + /// + public void Write(byte[] buffer, int offset, int count) + { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + if (ReadOnly) + throw new InvalidOperationException("Buffer is read-only"); + if ((long)_view.len > int.MaxValue) + throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported."); + if (count > buffer.Length) + throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer."); + if (count > (int)_view.len) + throw new ArgumentOutOfRangeException("count", "Count is bigger than the python buffer."); + if (_view.ndim != 1) + throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); + + Marshal.Copy(buffer, offset, _view.buf, count); + } + + /// + /// Reads the buffer of a python object into a managed byte array. This can be used to pass data like images from python to managed. + /// + public int Read(byte[] buffer, int offset, int count) { + if (disposedValue) + throw new ObjectDisposedException(nameof(PyBuffer)); + if (count > buffer.Length) + throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer."); + if (_view.ndim != 1) + throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); + if (_view.len.ToInt64() > int.MaxValue) + throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported."); + + int copylen = count < (int)_view.len ? count : (int)_view.len; + Marshal.Copy(_view.buf, buffer, offset, copylen); + return copylen; + } + + private bool disposedValue = false; // To detect redundant calls + + private void Dispose(bool disposing) + { + if (!disposedValue) + { + if (Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be initialized"); + + // this also decrements ref count for _view->obj + Runtime.PyBuffer_Release(ref _view); + + _exporter = null; + Shape = null; + Strides = null; + SubOffsets = null; + + disposedValue = true; + } + } + + ~PyBuffer() + { + if (disposedValue) + { + return; + } + Finalizer.Instance.AddFinalizedObject(ref _view.obj); + } + + /// + /// Release the buffer view and decrement the reference count for view->obj. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. + /// It is an error to call this function on a buffer that was not obtained via . + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 7237d1990..ade873f7a 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -29,9 +29,8 @@ public PyDict(IntPtr ptr) : base(ptr) /// /// Creates a new Python dictionary object. /// - public PyDict() + public PyDict() : base(Runtime.PyDict_New()) { - obj = Runtime.PyDict_New(); if (obj == IntPtr.Zero) { throw new PythonException(); @@ -47,14 +46,13 @@ public PyDict() /// ArgumentException will be thrown if the given object is not a /// Python dictionary object. /// - public PyDict(PyObject o) + public PyDict(PyObject o) : base(o.obj) { + Runtime.XIncref(o.obj); if (!IsDictType(o)) { throw new ArgumentException("object is not a dict"); } - Runtime.XIncref(o.obj); - obj = o.obj; } @@ -139,12 +137,20 @@ public PyObject Values() /// public PyObject Items() { - IntPtr items = Runtime.PyDict_Items(obj); - if (items == IntPtr.Zero) + var items = Runtime.PyDict_Items(this.obj); + try { - throw new PythonException(); + if (items.IsNull()) + { + throw new PythonException(); + } + + return items.MoveToPyObject(); + } + finally + { + items.Dispose(); } - return new PyObject(items); } diff --git a/src/runtime/pyfloat.cs b/src/runtime/pyfloat.cs index edfaca542..b07b95de1 100644 --- a/src/runtime/pyfloat.cs +++ b/src/runtime/pyfloat.cs @@ -4,7 +4,6 @@ namespace Python.Runtime { /// /// Represents a Python float object. See the documentation at - /// PY2: https://docs.python.org/2/c-api/float.html /// PY3: https://docs.python.org/3/c-api/float.html /// for details. /// @@ -31,14 +30,8 @@ public PyFloat(IntPtr ptr) : base(ptr) /// ArgumentException will be thrown if the given object is not a /// Python float object. /// - public PyFloat(PyObject o) + public PyFloat(PyObject o) : base(FromObject(o)) { - if (!IsFloatType(o)) - { - throw new ArgumentException("object is not a float"); - } - Runtime.XIncref(o.obj); - obj = o.obj; } @@ -48,12 +41,36 @@ public PyFloat(PyObject o) /// /// Creates a new Python float from a double value. /// - public PyFloat(double value) + public PyFloat(double value) : base(FromDouble(value)) + { + } + + private static IntPtr FromObject(PyObject o) + { + if (o == null || !IsFloatType(o)) + { + throw new ArgumentException("object is not a float"); + } + Runtime.XIncref(o.obj); + return o.obj; + } + + private static IntPtr FromDouble(double value) { - obj = Runtime.PyFloat_FromDouble(value); - Runtime.CheckExceptionOccurred(); + IntPtr val = Runtime.PyFloat_FromDouble(value); + PythonException.ThrowIfIsNull(val); + return val; } + private static IntPtr FromString(string value) + { + using (var s = new PyString(value)) + { + IntPtr val = Runtime.PyFloat_FromString(s.obj, IntPtr.Zero); + PythonException.ThrowIfIsNull(val); + return val; + } + } /// /// PyFloat Constructor @@ -61,13 +78,8 @@ public PyFloat(double value) /// /// Creates a new Python float from a string value. /// - public PyFloat(string value) + public PyFloat(string value) : base(FromString(value)) { - using (var s = new PyString(value)) - { - obj = Runtime.PyFloat_FromString(s.obj, IntPtr.Zero); - Runtime.CheckExceptionOccurred(); - } } @@ -94,7 +106,7 @@ public static bool IsFloatType(PyObject value) public static PyFloat AsFloat(PyObject value) { IntPtr op = Runtime.PyNumber_Float(value.obj); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(op); return new PyFloat(op); } } diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs index f6911d9d7..f8718cb95 100644 --- a/src/runtime/pyint.cs +++ b/src/runtime/pyint.cs @@ -31,16 +31,26 @@ public PyInt(IntPtr ptr) : base(ptr) /// ArgumentException will be thrown if the given object is not a /// Python int object. /// - public PyInt(PyObject o) + public PyInt(PyObject o) : base(FromObject(o)) { - if (!IsIntType(o)) + } + + private static IntPtr FromObject(PyObject o) + { + if (o == null || !IsIntType(o)) { throw new ArgumentException("object is not an int"); } Runtime.XIncref(o.obj); - obj = o.obj; + return o.obj; } + private static IntPtr FromInt(int value) + { + IntPtr val = Runtime.PyInt_FromInt32(value); + PythonException.ThrowIfIsNull(val); + return val; + } /// /// PyInt Constructor @@ -48,10 +58,8 @@ public PyInt(PyObject o) /// /// Creates a new Python int from an int32 value. /// - public PyInt(int value) + public PyInt(int value) : base(FromInt(value)) { - obj = Runtime.PyInt_FromInt32(value); - Runtime.CheckExceptionOccurred(); } @@ -61,11 +69,8 @@ public PyInt(int value) /// /// Creates a new Python int from a uint32 value. /// - [CLSCompliant(false)] - public PyInt(uint value) : base(IntPtr.Zero) + public PyInt(uint value) : base(FromLong(value)) { - obj = Runtime.PyInt_FromInt64(value); - Runtime.CheckExceptionOccurred(); } @@ -75,10 +80,15 @@ public PyInt(uint value) : base(IntPtr.Zero) /// /// Creates a new Python int from an int64 value. /// - public PyInt(long value) : base(IntPtr.Zero) + public PyInt(long value) : base(FromLong(value)) { - obj = Runtime.PyInt_FromInt64(value); - Runtime.CheckExceptionOccurred(); + } + + private static IntPtr FromLong(long value) + { + IntPtr val = Runtime.PyInt_FromInt64(value); + PythonException.ThrowIfIsNull(val); + return val; } @@ -88,11 +98,8 @@ public PyInt(long value) : base(IntPtr.Zero) /// /// Creates a new Python int from a uint64 value. /// - [CLSCompliant(false)] - public PyInt(ulong value) : base(IntPtr.Zero) + public PyInt(ulong value) : base(FromLong((long)value)) { - obj = Runtime.PyInt_FromInt64((long)value); - Runtime.CheckExceptionOccurred(); } @@ -113,7 +120,6 @@ public PyInt(short value) : this((int)value) /// /// Creates a new Python int from a uint16 value. /// - [CLSCompliant(false)] public PyInt(ushort value) : this((int)value) { } @@ -136,22 +142,26 @@ public PyInt(byte value) : this((int)value) /// /// Creates a new Python int from an sbyte value. /// - [CLSCompliant(false)] public PyInt(sbyte value) : this((int)value) { } + private static IntPtr FromString(string value) + { + IntPtr val = Runtime.PyInt_FromString(value, IntPtr.Zero, 0); + PythonException.ThrowIfIsNull(val); + return val; + } + /// /// PyInt Constructor /// /// /// Creates a new Python int from a string value. /// - public PyInt(string value) + public PyInt(string value) : base(FromString(value)) { - obj = Runtime.PyInt_FromString(value, IntPtr.Zero, 0); - Runtime.CheckExceptionOccurred(); } @@ -178,7 +188,7 @@ public static bool IsIntType(PyObject value) public static PyInt AsInt(PyObject value) { IntPtr op = Runtime.PyNumber_Int(value.obj); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(op); return new PyInt(op); } diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index ee07bcecf..2016ef4f8 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -9,7 +9,7 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/iterator.html /// for details. /// - public class PyIter : PyObject, IEnumerator + public class PyIter : PyObject, IEnumerator { private PyObject _current; @@ -26,57 +26,55 @@ public PyIter(IntPtr ptr) : base(ptr) } /// - /// PyIter Constructor + /// PyIter factory function. /// /// - /// Creates a Python iterator from an iterable. Like doing "iter(iterable)" in python. + /// Create a new PyIter from a given iterable. Like doing "iter(iterable)" in python. /// - public PyIter(PyObject iterable) + /// + /// + public static PyIter GetIter(PyObject iterable) { - obj = Runtime.PyObject_GetIter(iterable.obj); - if (obj == IntPtr.Zero) + if (iterable == null) { - throw new PythonException(); + throw new ArgumentNullException(); } + IntPtr val = Runtime.PyObject_GetIter(iterable.obj); + PythonException.ThrowIfIsNull(val); + return new PyIter(val); } protected override void Dispose(bool disposing) { - if (null != _current) - { - _current.Dispose(); - _current = null; - } + _current = null; base.Dispose(disposing); } public bool MoveNext() { - // dispose of the previous object, if there was one - if (null != _current) + NewReference next = Runtime.PyIter_Next(Reference); + if (next.IsNull()) { - _current.Dispose(); - _current = null; - } + if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } - IntPtr next = Runtime.PyIter_Next(obj); - if (next == IntPtr.Zero) - { + // stop holding the previous object, if there was one + _current = null; return false; } - _current = new PyObject(next); + _current = next.MoveToPyObject(); return true; } public void Reset() { - //Not supported in python. + throw new NotSupportedException(); } - public object Current - { - get { return _current; } - } + public PyObject Current => _current; + object System.Collections.IEnumerator.Current => _current; } } diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index b22d9d51f..7f5566401 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -22,6 +22,21 @@ public PyList(IntPtr ptr) : base(ptr) { } + /// + /// Creates new pointing to the same object, as the given reference. + /// + internal PyList(BorrowedReference reference) : base(reference) { } + + + private static IntPtr FromObject(PyObject o) + { + if (o == null || !IsListType(o)) + { + throw new ArgumentException("object is not a list"); + } + Runtime.XIncref(o.obj); + return o.obj; + } /// /// PyList Constructor @@ -31,14 +46,8 @@ public PyList(IntPtr ptr) : base(ptr) /// ArgumentException will be thrown if the given object is not a /// Python list object. /// - public PyList(PyObject o) + public PyList(PyObject o) : base(FromObject(o)) { - if (!IsListType(o)) - { - throw new ArgumentException("object is not a list"); - } - Runtime.XIncref(o.obj); - obj = o.obj; } @@ -48,38 +57,41 @@ public PyList(PyObject o) /// /// Creates a new empty Python list object. /// - public PyList() + public PyList() : base(Runtime.PyList_New(0)) { - obj = Runtime.PyList_New(0); if (obj == IntPtr.Zero) { throw new PythonException(); } } - - /// - /// PyList Constructor - /// - /// - /// Creates a new Python list object from an array of PyObjects. - /// - public PyList(PyObject[] items) + private static IntPtr FromArray(PyObject[] items) { int count = items.Length; - obj = Runtime.PyList_New(count); + IntPtr val = Runtime.PyList_New(count); for (var i = 0; i < count; i++) { IntPtr ptr = items[i].obj; Runtime.XIncref(ptr); - int r = Runtime.PyList_SetItem(obj, i, ptr); + int r = Runtime.PyList_SetItem(val, i, ptr); if (r < 0) { + Runtime.Py_DecRef(val); throw new PythonException(); } } + return val; } + /// + /// PyList Constructor + /// + /// + /// Creates a new Python list object from an array of PyObjects. + /// + public PyList(PyObject[] items) : base(FromArray(items)) + { + } /// /// IsListType Method @@ -120,7 +132,7 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(obj, item.obj); + int r = Runtime.PyList_Append(this.Reference, item.obj); if (r < 0) { throw new PythonException(); @@ -135,7 +147,7 @@ public void Append(PyObject item) /// public void Insert(int index, PyObject item) { - int r = Runtime.PyList_Insert(obj, index, item.obj); + int r = Runtime.PyList_Insert(this.Reference, index, item.obj); if (r < 0) { throw new PythonException(); @@ -151,7 +163,7 @@ public void Insert(int index, PyObject item) /// public void Reverse() { - int r = Runtime.PyList_Reverse(obj); + int r = Runtime.PyList_Reverse(this.Reference); if (r < 0) { throw new PythonException(); @@ -167,7 +179,7 @@ public void Reverse() /// public void Sort() { - int r = Runtime.PyList_Sort(obj); + int r = Runtime.PyList_Sort(this.Reference); if (r < 0) { throw new PythonException(); diff --git a/src/runtime/pylong.cs b/src/runtime/pylong.cs index 286af40df..0e21c7c49 100644 --- a/src/runtime/pylong.cs +++ b/src/runtime/pylong.cs @@ -31,16 +31,26 @@ public PyLong(IntPtr ptr) : base(ptr) /// ArgumentException will be thrown if the given object is not a /// Python long object. /// - public PyLong(PyObject o) + public PyLong(PyObject o) : base(FromObject(o)) { - if (!IsLongType(o)) + } + + private static IntPtr FromObject(PyObject o) + { + if (o == null || !IsLongType(o)) { throw new ArgumentException("object is not a long"); } Runtime.XIncref(o.obj); - obj = o.obj; + return o.obj; } + private static IntPtr FromInt(int value) + { + IntPtr val = Runtime.PyLong_FromLong(value); + PythonException.ThrowIfIsNull(val); + return val; + } /// /// PyLong Constructor @@ -48,10 +58,8 @@ public PyLong(PyObject o) /// /// Creates a new PyLong from an int32 value. /// - public PyLong(int value) + public PyLong(int value) : base(FromInt(value)) { - obj = Runtime.PyLong_FromLong(value); - Runtime.CheckExceptionOccurred(); } @@ -61,26 +69,34 @@ public PyLong(int value) /// /// Creates a new PyLong from a uint32 value. /// - [CLSCompliant(false)] - public PyLong(uint value) + public PyLong(uint value) : base(FromInt((int)value)) { - obj = Runtime.PyLong_FromLong(value); - Runtime.CheckExceptionOccurred(); } + internal static IntPtr FromLong(long value) + { + IntPtr val = Runtime.PyLong_FromLongLong(value); + PythonException.ThrowIfIsNull(val); + return val; + } + /// /// PyLong Constructor /// /// /// Creates a new PyLong from an int64 value. /// - public PyLong(long value) + public PyLong(long value) : base(FromLong(value)) { - obj = Runtime.PyLong_FromLongLong(value); - Runtime.CheckExceptionOccurred(); } + private static IntPtr FromULong(ulong value) + { + IntPtr val = Runtime.PyLong_FromUnsignedLongLong(value); + PythonException.ThrowIfIsNull(val); + return val; + } /// /// PyLong Constructor @@ -88,11 +104,8 @@ public PyLong(long value) /// /// Creates a new PyLong from a uint64 value. /// - [CLSCompliant(false)] - public PyLong(ulong value) + public PyLong(ulong value) : base(FromULong(value)) { - obj = Runtime.PyLong_FromUnsignedLongLong(value); - Runtime.CheckExceptionOccurred(); } @@ -102,10 +115,8 @@ public PyLong(ulong value) /// /// Creates a new PyLong from an int16 value. /// - public PyLong(short value) + public PyLong(short value) : base(FromInt((int)value)) { - obj = Runtime.PyLong_FromLong(value); - Runtime.CheckExceptionOccurred(); } @@ -115,11 +126,8 @@ public PyLong(short value) /// /// Creates a new PyLong from an uint16 value. /// - [CLSCompliant(false)] - public PyLong(ushort value) + public PyLong(ushort value) : base(FromInt((int)value)) { - obj = Runtime.PyLong_FromLong(value); - Runtime.CheckExceptionOccurred(); } @@ -129,10 +137,8 @@ public PyLong(ushort value) /// /// Creates a new PyLong from a byte value. /// - public PyLong(byte value) + public PyLong(byte value) : base(FromInt((int)value)) { - obj = Runtime.PyLong_FromLong(value); - Runtime.CheckExceptionOccurred(); } @@ -142,13 +148,16 @@ public PyLong(byte value) /// /// Creates a new PyLong from an sbyte value. /// - [CLSCompliant(false)] - public PyLong(sbyte value) + public PyLong(sbyte value) : base(FromInt((int)value)) { - obj = Runtime.PyLong_FromLong(value); - Runtime.CheckExceptionOccurred(); } + private static IntPtr FromDouble(double value) + { + IntPtr val = Runtime.PyLong_FromDouble(value); + PythonException.ThrowIfIsNull(val); + return val; + } /// /// PyLong Constructor @@ -156,12 +165,16 @@ public PyLong(sbyte value) /// /// Creates a new PyLong from an double value. /// - public PyLong(double value) + public PyLong(double value) : base(FromDouble(value)) { - obj = Runtime.PyLong_FromDouble(value); - Runtime.CheckExceptionOccurred(); } + private static IntPtr FromString(string value) + { + IntPtr val = Runtime.PyLong_FromString(value, IntPtr.Zero, 0); + PythonException.ThrowIfIsNull(val); + return val; + } /// /// PyLong Constructor @@ -169,10 +182,8 @@ public PyLong(double value) /// /// Creates a new PyLong from a string value. /// - public PyLong(string value) + public PyLong(string value) : base(FromString(value)) { - obj = Runtime.PyLong_FromString(value, IntPtr.Zero, 0); - Runtime.CheckExceptionOccurred(); } @@ -199,7 +210,7 @@ public static bool IsLongType(PyObject value) public static PyLong AsLong(PyObject value) { IntPtr op = Runtime.PyNumber_Long(value.obj); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(op); return new PyLong(op); } diff --git a/src/runtime/pynumber.cs b/src/runtime/pynumber.cs index 4f7373a8c..1af67b4e0 100644 --- a/src/runtime/pynumber.cs +++ b/src/runtime/pynumber.cs @@ -5,7 +5,6 @@ namespace Python.Runtime /// /// Represents a generic Python number. The methods of this class are /// equivalent to the Python "abstract number API". See - /// PY2: https://docs.python.org/2/c-api/number.html /// PY3: https://docs.python.org/3/c-api/number.html /// for details. /// @@ -18,11 +17,6 @@ protected PyNumber(IntPtr ptr) : base(ptr) { } - protected PyNumber() - { - } - - /// /// IsNumberType Method /// diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 47f413409..d68a9905b 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1,6 +1,9 @@ using System; using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; using System.Dynamic; +using System.Linq; using System.Linq.Expressions; namespace Python.Runtime @@ -12,10 +15,19 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IDisposable + [Serializable] + public partial class PyObject : DynamicObject, IEnumerable, IDisposable { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; private set; } +#endif + protected internal IntPtr obj = IntPtr.Zero; - private bool disposed = false; + + internal BorrowedReference Reference => new BorrowedReference(obj); /// /// PyObject Constructor @@ -28,22 +40,53 @@ public class PyObject : DynamicObject, IDisposable /// public PyObject(IntPtr ptr) { + if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); + obj = ptr; + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } - // Protected default constructor to allow subclasses to manage - // initialization in different ways as appropriate. + [Obsolete("for testing purposes only")] + internal PyObject(IntPtr ptr, bool skipCollect) + { + if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); - protected PyObject() + obj = ptr; + if (!skipCollect) + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif + } + + /// + /// Creates new pointing to the same object as + /// the . Increments refcount, allowing + /// to have ownership over its own reference. + /// + internal PyObject(BorrowedReference reference) { + if (reference.IsNull) throw new ArgumentNullException(nameof(reference)); + + obj = Runtime.SelfIncRef(reference.DangerousGetAddress()); + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. - ~PyObject() { - Dispose(); + if (obj == IntPtr.Zero) + { + return; + } + Finalizer.Instance.AddFinalizedObject(ref obj); } @@ -61,7 +104,8 @@ public IntPtr Handle /// - /// FromManagedObject Method + /// Gets raw Python proxy for this object (bypasses all conversions, + /// except null <==> None) /// /// /// Given an arbitrary managed object, return a Python instance that @@ -90,13 +134,29 @@ public static PyObject FromManagedObject(object ob) public object AsManagedObject(Type t) { object result; - if (!Converter.ToManaged(obj, t, out result, false)) + if (!Converter.ToManaged(obj, t, out result, true)) { - throw new InvalidCastException("cannot convert object to target type"); + throw new InvalidCastException("cannot convert object to target type", new PythonException()); } return result; } + /// + /// As Method + /// + /// + /// Return a managed object of the given type, based on the + /// value of the Python object. + /// + public T As() + { + if (typeof(T) == typeof(PyObject) || typeof(T) == typeof(object)) + { + return (T)(this as object); + } + return (T)AsManagedObject(typeof(T)); + } + /// /// Dispose Method @@ -111,17 +171,45 @@ public object AsManagedObject(Type t) /// protected virtual void Dispose(bool disposing) { - if (!disposed) + if (this.obj == IntPtr.Zero) + { + return; + } + + if (Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be initialized"); + + if (!Runtime.IsFinalizing) { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + long refcount = Runtime.Refcount(this.obj); + Debug.Assert(refcount > 0, "Object refcount is 0 or less"); + + if (refcount == 1) + { + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + + try + { + Runtime.XDecref(this.obj); + Runtime.CheckExceptionOccurred(); + } + finally + { + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType, errVal, traceback); + } + } + else { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(obj); - obj = IntPtr.Zero; - PythonEngine.ReleaseLock(gs); + Runtime.XDecref(this.obj); } - disposed = true; } + else + { + throw new InvalidOperationException("Runtime is already finalizing"); + } + this.obj = IntPtr.Zero; } public void Dispose() @@ -130,7 +218,6 @@ public void Dispose() GC.SuppressFinalize(this); } - /// /// GetPythonType Method /// @@ -154,6 +241,8 @@ public PyObject GetPythonType() /// public bool TypeCheck(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + return Runtime.PyObject_TypeCheck(obj, typeOrClass.obj); } @@ -166,6 +255,8 @@ public bool TypeCheck(PyObject typeOrClass) /// public bool HasAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + return Runtime.PyObject_HasAttrString(obj, name) != 0; } @@ -179,6 +270,8 @@ public bool HasAttr(string name) /// public bool HasAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + return Runtime.PyObject_HasAttr(obj, name.obj) != 0; } @@ -192,6 +285,8 @@ public bool HasAttr(PyObject name) /// public PyObject GetAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { @@ -202,7 +297,7 @@ public PyObject GetAttr(string name) /// - /// GetAttr Method + /// GetAttr Method. Returns fallback value if getting attribute fails for any reason. /// /// /// Returns the named attribute of the Python object, or the given @@ -210,6 +305,8 @@ public PyObject GetAttr(string name) /// public PyObject GetAttr(string name, PyObject _default) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { @@ -230,6 +327,8 @@ public PyObject GetAttr(string name, PyObject _default) /// public PyObject GetAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { @@ -249,6 +348,8 @@ public PyObject GetAttr(PyObject name) /// public PyObject GetAttr(PyObject name, PyObject _default) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { @@ -268,6 +369,9 @@ public PyObject GetAttr(PyObject name, PyObject _default) /// public void SetAttr(string name, PyObject value) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetAttrString(obj, name, value.obj); if (r < 0) { @@ -286,6 +390,9 @@ public void SetAttr(string name, PyObject value) /// public void SetAttr(PyObject name, PyObject value) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetAttr(obj, name.obj, value.obj); if (r < 0) { @@ -303,6 +410,8 @@ public void SetAttr(PyObject name, PyObject value) /// public void DelAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + int r = Runtime.PyObject_SetAttrString(obj, name, IntPtr.Zero); if (r < 0) { @@ -321,6 +430,8 @@ public void DelAttr(string name) /// public void DelAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + int r = Runtime.PyObject_SetAttr(obj, name.obj, IntPtr.Zero); if (r < 0) { @@ -339,6 +450,8 @@ public void DelAttr(PyObject name) /// public virtual PyObject GetItem(PyObject key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + IntPtr op = Runtime.PyObject_GetItem(obj, key.obj); if (op == IntPtr.Zero) { @@ -358,6 +471,8 @@ public virtual PyObject GetItem(PyObject key) /// public virtual PyObject GetItem(string key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + using (var pyKey = new PyString(key)) { return GetItem(pyKey); @@ -392,6 +507,9 @@ public virtual PyObject GetItem(int index) /// public virtual void SetItem(PyObject key, PyObject value) { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetItem(obj, key.obj, value.obj); if (r < 0) { @@ -410,6 +528,9 @@ public virtual void SetItem(PyObject key, PyObject value) /// public virtual void SetItem(string key, PyObject value) { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + using (var pyKey = new PyString(key)) { SetItem(pyKey, value); @@ -427,6 +548,8 @@ public virtual void SetItem(string key, PyObject value) /// public virtual void SetItem(int index, PyObject value) { + if (value == null) throw new ArgumentNullException(nameof(value)); + using (var pyindex = new PyInt(index)) { SetItem(pyindex, value); @@ -444,6 +567,8 @@ public virtual void SetItem(int index, PyObject value) /// public virtual void DelItem(PyObject key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + int r = Runtime.PyObject_DelItem(obj, key.obj); if (r < 0) { @@ -462,6 +587,8 @@ public virtual void DelItem(PyObject key) /// public virtual void DelItem(string key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + using (var pyKey = new PyString(key)) { DelItem(pyKey); @@ -493,9 +620,9 @@ public virtual void DelItem(int index) /// Returns the length for objects that support the Python sequence /// protocol, or 0 if the object does not support the protocol. /// - public virtual int Length() + public virtual long Length() { - int s = Runtime.PyObject_Size(obj); + var s = Runtime.PyObject_Size(obj); if (s < 0) { Runtime.PyErr_Clear(); @@ -573,10 +700,11 @@ public PyObject GetIterator() /// python object to be iterated over in C#. A PythonException will be /// raised if the object is not iterable. /// - public IEnumerator GetEnumerator() + public IEnumerator GetEnumerator() { - return new PyIter(this); + return PyIter.GetIter(this); } + IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); /// @@ -584,10 +712,13 @@ public IEnumerator GetEnumerator() /// /// /// Invoke the callable object with the given arguments, passed as a - /// PyObject[]. A PythonException is raised if the invokation fails. + /// PyObject[]. A PythonException is raised if the invocation fails. /// public PyObject Invoke(params PyObject[] args) { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + var t = new PyTuple(args); IntPtr r = Runtime.PyObject_Call(obj, t.obj, IntPtr.Zero); t.Dispose(); @@ -604,10 +735,12 @@ public PyObject Invoke(params PyObject[] args) /// /// /// Invoke the callable object with the given arguments, passed as a - /// Python tuple. A PythonException is raised if the invokation fails. + /// Python tuple. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyTuple args) { + if (args == null) throw new ArgumentNullException(nameof(args)); + IntPtr r = Runtime.PyObject_Call(obj, args.obj, IntPtr.Zero); if (r == IntPtr.Zero) { @@ -622,12 +755,15 @@ public PyObject Invoke(PyTuple args) /// /// /// Invoke the callable object with the given positional and keyword - /// arguments. A PythonException is raised if the invokation fails. + /// arguments. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyObject[] args, PyDict kw) { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + var t = new PyTuple(args); - IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw != null ? kw.obj : IntPtr.Zero); + IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw?.obj ?? IntPtr.Zero); t.Dispose(); if (r == IntPtr.Zero) { @@ -642,11 +778,13 @@ public PyObject Invoke(PyObject[] args, PyDict kw) /// /// /// Invoke the callable object with the given positional and keyword - /// arguments. A PythonException is raised if the invokation fails. + /// arguments. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyTuple args, PyDict kw) { - IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw != null ? kw.obj : IntPtr.Zero); + if (args == null) throw new ArgumentNullException(nameof(args)); + + IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw?.obj ?? IntPtr.Zero); if (r == IntPtr.Zero) { throw new PythonException(); @@ -660,10 +798,14 @@ public PyObject Invoke(PyTuple args, PyDict kw) /// /// /// Invoke the named method of the object with the given arguments. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, params PyObject[] args) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args); method.Dispose(); @@ -676,10 +818,51 @@ public PyObject InvokeMethod(string name, params PyObject[] args) /// /// /// Invoke the named method of the object with the given arguments. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyTuple args) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, params PyObject[] args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, PyTuple args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args); method.Dispose(); @@ -693,10 +876,14 @@ public PyObject InvokeMethod(string name, PyTuple args) /// /// Invoke the named method of the object with the given arguments /// and keyword arguments. Keyword args are passed as a PyDict object. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyObject[] args, PyDict kw) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args, kw); method.Dispose(); @@ -710,10 +897,13 @@ public PyObject InvokeMethod(string name, PyObject[] args, PyDict kw) /// /// Invoke the named method of the object with the given arguments /// and keyword arguments. Keyword args are passed as a PyDict object. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyTuple args, PyDict kw) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args, kw); method.Dispose(); @@ -730,6 +920,8 @@ public PyObject InvokeMethod(string name, PyTuple args, PyDict kw) /// public bool IsInstance(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + int r = Runtime.PyObject_IsInstance(obj, typeOrClass.obj); if (r < 0) { @@ -749,6 +941,8 @@ public bool IsInstance(PyObject typeOrClass) /// public bool IsSubclass(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + int r = Runtime.PyObject_IsSubclass(obj, typeOrClass.obj); if (r < 0) { @@ -781,7 +975,7 @@ public bool IsCallable() /// public bool IsIterable() { - return Runtime.PyIter_Check(obj); + return Runtime.PyObject_IsIterable(obj); } @@ -797,6 +991,10 @@ public bool IsTrue() return Runtime.PyObject_IsTrue(obj) != 0; } + /// + /// Return true if the object is None + /// + public bool IsNone() => CheckNone(this) == null; /// /// Dir Method @@ -884,7 +1082,20 @@ public override bool Equals(object o) /// public override int GetHashCode() { - return Runtime.PyObject_Hash(obj).ToInt32(); + return ((ulong)Runtime.PyObject_Hash(obj)).GetHashCode(); + } + + /// + /// GetBuffer Method. This Method only works for objects that have a buffer (like "bytes", "bytearray" or "array.array") + /// + /// + /// Send a request to the PyObject to fill in view as specified by flags. If the PyObject cannot provide a buffer of the exact type, it MUST raise PyExc_BufferError, set view->obj to NULL and return -1. + /// On success, fill in view, set view->obj to a new reference to exporter and return 0. In the case of chained buffer providers that redirect requests to a single object, view->obj MAY refer to this object instead of exporter(See Buffer Object Structures). + /// Successful calls to must be paired with calls to , similar to malloc() and free(). Thus, after the consumer is done with the buffer, must be called exactly once. + /// + public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE) + { + return new PyBuffer(this, flags); } @@ -915,6 +1126,34 @@ public override bool TrySetMember(SetMemberBinder binder, object value) return true; } + private void GetArgs(object[] inargs, CallInfo callInfo, out PyTuple args, out PyDict kwargs) + { + if (callInfo == null || callInfo.ArgumentNames.Count == 0) + { + GetArgs(inargs, out args, out kwargs); + return; + } + + // Support for .net named arguments + var namedArgumentCount = callInfo.ArgumentNames.Count; + var regularArgumentCount = callInfo.ArgumentCount - namedArgumentCount; + + var argTuple = Runtime.PyTuple_New(regularArgumentCount); + for (int i = 0; i < regularArgumentCount; ++i) + { + AddArgument(argTuple, i, inargs[i]); + } + args = new PyTuple(argTuple); + + var namedArgs = new object[namedArgumentCount * 2]; + for (int i = 0; i < namedArgumentCount; ++i) + { + namedArgs[i * 2] = callInfo.ArgumentNames[i]; + namedArgs[i * 2 + 1] = inargs[regularArgumentCount + i]; + } + kwargs = Py.kw(namedArgs); + } + private void GetArgs(object[] inargs, out PyTuple args, out PyDict kwargs) { int arg_count; @@ -925,22 +1164,10 @@ private void GetArgs(object[] inargs, out PyTuple args, out PyDict kwargs) IntPtr argtuple = Runtime.PyTuple_New(arg_count); for (var i = 0; i < arg_count; i++) { - IntPtr ptr; - if (inargs[i] is PyObject) - { - ptr = ((PyObject)inargs[i]).Handle; - Runtime.XIncref(ptr); - } - else - { - ptr = Converter.ToPython(inargs[i], inargs[i]?.GetType()); - } - if (Runtime.PyTuple_SetItem(argtuple, i, ptr) < 0) - { - throw new PythonException(); - } + AddArgument(argtuple, i, inargs[i]); } args = new PyTuple(argtuple); + kwargs = null; for (int i = arg_count; i < inargs.Length; i++) { @@ -959,6 +1186,32 @@ private void GetArgs(object[] inargs, out PyTuple args, out PyDict kwargs) } } + private static void AddArgument(IntPtr argtuple, int i, object target) + { + IntPtr ptr = GetPythonObject(target); + + if (Runtime.PyTuple_SetItem(argtuple, i, ptr) < 0) + { + throw new PythonException(); + } + } + + private static IntPtr GetPythonObject(object target) + { + IntPtr ptr; + if (target is PyObject) + { + ptr = ((PyObject)target).Handle; + Runtime.XIncref(ptr); + } + else + { + ptr = Converter.ToPython(target, target?.GetType()); + } + + return ptr; + } + public override bool TryInvokeMember(InvokeMemberBinder binder, object[] args, out object result) { if (this.HasAttr(binder.Name) && this.GetAttr(binder.Name).IsCallable()) @@ -967,7 +1220,7 @@ public override bool TryInvokeMember(InvokeMemberBinder binder, object[] args, o PyDict kwargs = null; try { - GetArgs(args, out pyargs, out kwargs); + GetArgs(args, binder.CallInfo, out pyargs, out kwargs); result = CheckNone(InvokeMethod(binder.Name, pyargs, kwargs)); } finally @@ -997,7 +1250,7 @@ public override bool TryInvoke(InvokeBinder binder, object[] args, out object re PyDict kwargs = null; try { - GetArgs(args, out pyargs, out kwargs); + GetArgs(args, binder.CallInfo, out pyargs, out kwargs); result = CheckNone(Invoke(pyargs, kwargs)); } finally @@ -1053,10 +1306,10 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceMultiply(this.obj, ((PyObject)arg).obj); break; case ExpressionType.Divide: - res = Runtime.PyNumber_Divide(this.obj, ((PyObject)arg).obj); + res = Runtime.PyNumber_TrueDivide(this.obj, ((PyObject)arg).obj); break; case ExpressionType.DivideAssign: - res = Runtime.PyNumber_InPlaceDivide(this.obj, ((PyObject)arg).obj); + res = Runtime.PyNumber_InPlaceTrueDivide(this.obj, ((PyObject)arg).obj); break; case ExpressionType.And: res = Runtime.PyNumber_And(this.obj, ((PyObject)arg).obj); @@ -1171,5 +1424,20 @@ public override bool TryUnaryOperation(UnaryOperationBinder binder, out object r result = CheckNone(new PyObject(res)); return true; } + + /// + /// Returns the enumeration of all dynamic member names. + /// + /// + /// This method exists for debugging purposes only. + /// + /// A sequence that contains dynamic member names. + public override IEnumerable GetDynamicMemberNames() + { + foreach (PyObject pyObj in Dir()) + { + yield return pyObj.ToString(); + } + } } } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs new file mode 100644 index 000000000..d61573733 --- /dev/null +++ b/src/runtime/pyscope.cs @@ -0,0 +1,660 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using System.Dynamic; + +namespace Python.Runtime +{ + public class PyScopeException : Exception + { + public PyScopeException(string message) + : base(message) + { + + } + } + + /// + /// Classes/methods have this attribute must be used with GIL obtained. + /// + public class PyGILAttribute : Attribute + { + } + + [PyGIL] + public class PyScope : DynamicObject, IDisposable + { + public readonly string Name; + + /// + /// the python Module object the scope associated with. + /// + internal IntPtr obj; + + /// + /// the variable dict of the scope. + /// + internal readonly IntPtr variables; + + private bool _isDisposed; + private bool _finalized = false; + + /// + /// The Manager this scope associated with. + /// It provides scopes this scope can import. + /// + internal readonly PyScopeManager Manager; + + /// + /// event which will be triggered after the scope disposed. + /// + public event Action OnDispose; + + /// + /// Constructor + /// + /// + /// Create a scope based on a Python Module. + /// + internal PyScope(IntPtr ptr, PyScopeManager manager) + { + if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(ptr), Runtime.PyModuleType)) + { + throw new PyScopeException("object is not a module"); + } + Manager = manager ?? PyScopeManager.Global; + obj = ptr; + //Refcount of the variables not increase + variables = Runtime.PyModule_GetDict(obj); + PythonException.ThrowIfIsNull(variables); + + int res = Runtime.PyDict_SetItem( + variables, PyIdentifier.__builtins__, + Runtime.PyEval_GetBuiltins() + ); + PythonException.ThrowIfIsNotZero(res); + this.Name = this.Get("__name__"); + } + + /// + /// return the variable dict of the scope. + /// + /// + public PyDict Variables() + { + Runtime.XIncref(variables); + return new PyDict(variables); + } + + /// + /// Create a scope, and import all from this scope + /// + /// + public PyScope NewScope() + { + var scope = Manager.Create(); + scope.ImportAll(this); + return scope; + } + + /// + /// Import method + /// + /// + /// Import a scope or a module of given name, + /// scope will be looked up first. + /// + public dynamic Import(string name, string asname = null) + { + Check(); + if (String.IsNullOrEmpty(asname)) + { + asname = name; + } + PyScope scope; + Manager.TryGet(name, out scope); + if (scope != null) + { + Import(scope, asname); + return scope; + } + else + { + PyObject module = PythonEngine.ImportModule(name); + Import(module, asname); + return module; + } + } + + /// + /// Import method + /// + /// + /// Import a scope as a variable of given name. + /// + public void Import(PyScope scope, string asname) + { + this.Set(asname, scope.obj); + } + + /// + /// Import Method + /// + /// + /// The 'import .. as ..' statement in Python. + /// Import a module as a variable into the scope. + /// + public void Import(PyObject module, string asname = null) + { + if (String.IsNullOrEmpty(asname)) + { + asname = module.GetAttr("__name__").As(); + } + Set(asname, module); + } + + /// + /// ImportAll Method + /// + /// + /// The 'import * from ..' statement in Python. + /// Import all content of a scope/module of given name into the scope, scope will be looked up first. + /// + public void ImportAll(string name) + { + PyScope scope; + Manager.TryGet(name, out scope); + if (scope != null) + { + ImportAll(scope); + return; + } + else + { + PyObject module = PythonEngine.ImportModule(name); + ImportAll(module); + } + } + + /// + /// ImportAll Method + /// + /// + /// Import all variables of the scope into this scope. + /// + public void ImportAll(PyScope scope) + { + int result = Runtime.PyDict_Update(variables, scope.variables); + if (result < 0) + { + throw new PythonException(); + } + } + + /// + /// ImportAll Method + /// + /// + /// Import all variables of the module into this scope. + /// + public void ImportAll(PyObject module) + { + if (Runtime.PyObject_Type(module.obj) != Runtime.PyModuleType) + { + throw new PyScopeException("object is not a module"); + } + var module_dict = Runtime.PyModule_GetDict(module.obj); + int result = Runtime.PyDict_Update(variables, module_dict); + if (result < 0) + { + throw new PythonException(); + } + } + + /// + /// ImportAll Method + /// + /// + /// Import all variables in the dictionary into this scope. + /// + public void ImportAll(PyDict dict) + { + int result = Runtime.PyDict_Update(variables, dict.obj); + if (result < 0) + { + throw new PythonException(); + } + } + + /// + /// Execute method + /// + /// + /// Execute a Python ast and return the result as a PyObject. + /// The ast can be either an expression or stmts. + /// + public PyObject Execute(PyObject script, PyDict locals = null) + { + Check(); + IntPtr _locals = locals == null ? variables : locals.obj; + IntPtr ptr = Runtime.PyEval_EvalCode(script.Handle, variables, _locals); + PythonException.ThrowIfIsNull(ptr); + if (ptr == Runtime.PyNone) + { + Runtime.XDecref(ptr); + return null; + } + return new PyObject(ptr); + } + + /// + /// Execute method + /// + /// + /// Execute a Python ast and return the result as a PyObject, + /// and convert the result to a Managed Object of given type. + /// The ast can be either an expression or stmts. + /// + public T Execute(PyObject script, PyDict locals = null) + { + Check(); + PyObject pyObj = Execute(script, locals); + if (pyObj == null) + { + return default(T); + } + var obj = pyObj.As(); + return obj; + } + + /// + /// Eval method + /// + /// + /// Evaluate a Python expression and return the result as a PyObject + /// or null if an exception is raised. + /// + public PyObject Eval(string code, PyDict locals = null) + { + Check(); + IntPtr _locals = locals == null ? variables : locals.obj; + + NewReference reference = Runtime.PyRun_String( + code, RunFlagType.Eval, variables, _locals + ); + PythonException.ThrowIfIsNull(reference); + return reference.MoveToPyObject(); + } + + /// + /// Evaluate a Python expression + /// + /// + /// Evaluate a Python expression + /// and convert the result to a Managed Object of given type. + /// + public T Eval(string code, PyDict locals = null) + { + Check(); + PyObject pyObj = Eval(code, locals); + var obj = pyObj.As(); + return obj; + } + + /// + /// Exec Method + /// + /// + /// Exec a Python script and save its local variables in the current local variable dict. + /// + public void Exec(string code, PyDict locals = null) + { + Check(); + IntPtr _locals = locals == null ? variables : locals.obj; + Exec(code, variables, _locals); + } + + private void Exec(string code, IntPtr _globals, IntPtr _locals) + { + NewReference reference = Runtime.PyRun_String( + code, RunFlagType.File, _globals, _locals + ); + PythonException.ThrowIfIsNull(reference); + reference.Dispose(); + } + + /// + /// Set Variable Method + /// + /// + /// Add a new variable to the variables dict if it not exist + /// or update its value if the variable exists. + /// + public void Set(string name, object value) + { + IntPtr _value = Converter.ToPython(value, value?.GetType()); + Set(name, _value); + Runtime.XDecref(_value); + } + + private void Set(string name, IntPtr value) + { + Check(); + using (var pyKey = new PyString(name)) + { + int r = Runtime.PyObject_SetItem(variables, pyKey.obj, value); + if (r < 0) + { + throw new PythonException(); + } + } + } + + /// + /// Remove Method + /// + /// + /// Remove a variable from the variables dict. + /// + public void Remove(string name) + { + Check(); + using (var pyKey = new PyString(name)) + { + int r = Runtime.PyObject_DelItem(variables, pyKey.obj); + if (r < 0) + { + throw new PythonException(); + } + } + } + + /// + /// Contains Method + /// + /// + /// Returns true if the variable exists in the scope. + /// + public bool Contains(string name) + { + Check(); + using (var pyKey = new PyString(name)) + { + return Runtime.PyMapping_HasKey(variables, pyKey.obj) != 0; + } + } + + /// + /// Get Method + /// + /// + /// Returns the value of the variable of given name. + /// If the variable does not exist, throw an Exception. + /// + public PyObject Get(string name) + { + PyObject scope; + var state = TryGet(name, out scope); + if (!state) + { + throw new PyScopeException($"The scope of name '{Name}' has no attribute '{name}'"); + } + return scope; + } + + /// + /// TryGet Method + /// + /// + /// Returns the value of the variable, local variable first. + /// If the variable does not exist, return null. + /// + public bool TryGet(string name, out PyObject value) + { + Check(); + using (var pyKey = new PyString(name)) + { + if (Runtime.PyMapping_HasKey(variables, pyKey.obj) != 0) + { + IntPtr op = Runtime.PyObject_GetItem(variables, pyKey.obj); + if (op == IntPtr.Zero) + { + throw new PythonException(); + } + if (op == Runtime.PyNone) + { + Runtime.XDecref(op); + value = null; + return true; + } + value = new PyObject(op); + return true; + } + else + { + value = null; + return false; + } + } + } + + /// + /// Get Method + /// + /// + /// Obtain the value of the variable of given name, + /// and convert the result to a Managed Object of given type. + /// If the variable does not exist, throw an Exception. + /// + public T Get(string name) + { + Check(); + PyObject pyObj = Get(name); + if (pyObj == null) + { + return default(T); + } + return pyObj.As(); + } + + /// + /// TryGet Method + /// + /// + /// Obtain the value of the variable of given name, + /// and convert the result to a Managed Object of given type. + /// If the variable does not exist, return false. + /// + public bool TryGet(string name, out T value) + { + Check(); + PyObject pyObj; + var result = TryGet(name, out pyObj); + if (!result) + { + value = default(T); + return false; + } + if (pyObj == null) + { + if (typeof(T).IsValueType) + { + throw new PyScopeException($"The value of the attribute '{name}' is None which cannot be convert to '{typeof(T).ToString()}'"); + } + else + { + value = default(T); + return true; + } + } + value = pyObj.As(); + return true; + } + + public override bool TryGetMember(GetMemberBinder binder, out object result) + { + result = this.Get(binder.Name); + return true; + } + + public override bool TrySetMember(SetMemberBinder binder, object value) + { + this.Set(binder.Name, value); + return true; + } + + private void Check() + { + if (_isDisposed) + { + throw new PyScopeException($"The scope of name '{Name}' object has been disposed"); + } + } + + public void Dispose() + { + if (_isDisposed) + { + return; + } + _isDisposed = true; + Runtime.XDecref(obj); + this.OnDispose?.Invoke(this); + } + + ~PyScope() + { + if (_finalized || _isDisposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(ref obj); + } + } + + public class PyScopeManager + { + public static PyScopeManager Global; + + private Dictionary NamedScopes = new Dictionary(); + + internal static void Reset() + { + Global = new PyScopeManager(); + } + + internal PyScope NewScope(string name) + { + if (name == null) + { + name = ""; + } + var module = Runtime.PyModule_New(name); + if (module == IntPtr.Zero) + { + throw new PythonException(); + } + return new PyScope(module, this); + } + + /// + /// Create Method + /// + /// + /// Create an anonymous scope. + /// + [PyGIL] + public PyScope Create() + { + var scope = this.NewScope(null); + return scope; + } + + /// + /// Create Method + /// + /// + /// Create an named scope of given name. + /// + [PyGIL] + public PyScope Create(string name) + { + if (String.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name)); + } + if (name != null && Contains(name)) + { + throw new PyScopeException($"A scope of name '{name}' does already exist"); + } + var scope = this.NewScope(name); + scope.OnDispose += Remove; + NamedScopes[name] = scope; + return scope; + } + + /// + /// Contains Method + /// + /// + /// return true if the scope exists in this manager. + /// + public bool Contains(string name) + { + return NamedScopes.ContainsKey(name); + } + + /// + /// Get Method + /// + /// + /// Find the scope in this manager. + /// If the scope not exist, an Exception will be thrown. + /// + public PyScope Get(string name) + { + if (String.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name)); + } + if (NamedScopes.ContainsKey(name)) + { + return NamedScopes[name]; + } + throw new PyScopeException($"There is no scope named '{name}' registered in this manager"); + } + + /// + /// Get Method + /// + /// + /// Try to find the scope in this manager. + /// + public bool TryGet(string name, out PyScope scope) + { + return NamedScopes.TryGetValue(name, out scope); + } + + /// + /// Remove Method + /// + /// + /// remove the scope from this manager. + /// + public void Remove(PyScope scope) + { + NamedScopes.Remove(scope.Name); + } + + [PyGIL] + public void Clear() + { + var scopes = NamedScopes.Values.ToList(); + foreach (var scope in scopes) + { + scope.Dispose(); + } + } + } +} diff --git a/src/runtime/pysequence.cs b/src/runtime/pysequence.cs index bfaee79a6..1850ef7de 100644 --- a/src/runtime/pysequence.cs +++ b/src/runtime/pysequence.cs @@ -16,9 +16,7 @@ protected PySequence(IntPtr ptr) : base(ptr) { } - protected PySequence() - { - } + internal PySequence(BorrowedReference reference) : base(reference) { } /// diff --git a/src/runtime/pystring.cs b/src/runtime/pystring.cs index c9c4f9f5b..b3d0dc86d 100644 --- a/src/runtime/pystring.cs +++ b/src/runtime/pystring.cs @@ -26,6 +26,16 @@ public PyString(IntPtr ptr) : base(ptr) } + private static IntPtr FromObject(PyObject o) + { + if (o == null || !IsStringType(o)) + { + throw new ArgumentException("object is not a string"); + } + Runtime.XIncref(o.obj); + return o.obj; + } + /// /// PyString Constructor /// @@ -34,27 +44,25 @@ public PyString(IntPtr ptr) : base(ptr) /// An ArgumentException will be thrown if the given object is not /// a Python string object. /// - public PyString(PyObject o) + public PyString(PyObject o) : base(FromObject(o)) { - if (!IsStringType(o)) - { - throw new ArgumentException("object is not a string"); - } - Runtime.XIncref(o.obj); - obj = o.obj; } + private static IntPtr FromString(string s) + { + IntPtr val = Runtime.PyUnicode_FromUnicode(s, s.Length); + PythonException.ThrowIfIsNull(val); + return val; + } /// /// PyString Constructor /// /// /// Creates a Python string from a managed string. /// - public PyString(string s) + public PyString(string s) : base(FromString(s)) { - obj = Runtime.PyUnicode_FromUnicode(s, s.Length); - Runtime.CheckExceptionOccurred(); } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 1fd3b239a..df6cf7101 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,6 +12,14 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { + public static ShutdownMode ShutdownMode + { + get => Runtime.ShutdownMode; + set => Runtime.ShutdownMode = value; + } + + public static ShutdownMode DefaultShutdownMode => Runtime.GetDefaultShutdownMode(); + private static DelegateManager delegateManager; private static bool initialized; private static IntPtr _pythonHome = IntPtr.Zero; @@ -80,6 +88,7 @@ public static string PythonHome } set { + // this value is null in the beginning Marshal.FreeHGlobal(_pythonHome); _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); Runtime.Py_SetPythonHome(_pythonHome); @@ -95,10 +104,6 @@ public static string PythonPath } set { - if (Runtime.IsPython2) - { - throw new NotSupportedException("Set PythonPath not supported on Python 2"); - } Marshal.FreeHGlobal(_pythonPath); _pythonPath = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); Runtime.Py_SetPath(_pythonPath); @@ -130,6 +135,16 @@ public static string Compiler get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCompiler()); } } + /// + /// Set the NoSiteFlag to disable loading the site module. + /// Must be called before Initialize. + /// https://docs.python.org/3/c-api/init.html#c.Py_NoSiteFlag + /// + public static void SetNoSiteFlag() + { + Runtime.SetNoSiteFlag(); + } + public static int RunSimpleString(string code) { return Runtime.PyRun_SimpleString(code); @@ -140,9 +155,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); } /// @@ -153,74 +168,93 @@ public static void Initialize(bool setSysArgv = true) /// more than once, though initialization will only happen on the /// first call. It is *not* necessary to hold the Python global /// interpreter lock (GIL) to call this method. + /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - if (!initialized) + if (initialized) { - // Creating the delegateManager MUST happen before Runtime.Initialize - // is called. If it happens afterwards, DelegateManager's CodeGenerator - // throws an exception in its ctor. This exception is eaten somehow - // during an initial "import clr", and the world ends shortly thereafter. - // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). - delegateManager = new DelegateManager(); - Runtime.Initialize(); - initialized = true; - Exceptions.Clear(); - - if (setSysArgv) - { - Py.SetArgv(args); - } + return; + } + // Creating the delegateManager MUST happen before Runtime.Initialize + // is called. If it happens afterwards, DelegateManager's CodeGenerator + // throws an exception in its ctor. This exception is eaten somehow + // during an initial "import clr", and the world ends shortly thereafter. + // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). + delegateManager = new DelegateManager(); + Runtime.Initialize(initSigs, mode); + initialized = true; + Exceptions.Clear(); + + // Make sure we clean up properly on app domain unload. + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + + // The global scope gets used implicitly quite early on, remember + // to clear it out when we shut down. + AddShutdownHandler(PyScopeManager.Global.Clear); + + if (setSysArgv) + { + Py.SetArgv(args); + } + if (mode == ShutdownMode.Normal) + { + // TOOD: Check if this can be remove completely or not. // register the atexit callback (this doesn't use Py_AtExit as the C atexit // callbacks are called after python is fully finalized but the python ones // are called while the python engine is still running). - string code = - "import atexit, clr\n" + - "atexit.register(clr._AtExit)\n"; - PythonEngine.Exec(code); + //string code = + // "import atexit, clr\n" + + // "atexit.register(clr._AtExit)\n"; + //PythonEngine.Exec(code); + } - // Load the clr.py resource into the clr module - IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); - IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + // Load the clr.py resource into the clr module + IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); + IntPtr clr_dict = Runtime.PyModule_GetDict(clr); - var locals = new PyDict(); - try + var locals = new PyDict(); + try + { + IntPtr module = Runtime.PyImport_AddModule("clr._extras"); + IntPtr module_globals = Runtime.PyModule_GetDict(module); + IntPtr builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + + Assembly assembly = Assembly.GetExecutingAssembly(); + using (Stream stream = assembly.GetManifestResourceStream("clr.py")) + using (var reader = new StreamReader(stream)) { - IntPtr module = Runtime.PyImport_AddModule("clr._extras"); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); - - Assembly assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("clr.py")) - using (var reader = new StreamReader(stream)) - { - // add the contents of clr.py to the module - string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Handle); - } + // add the contents of clr.py to the module + string clr_py = reader.ReadToEnd(); + Exec(clr_py, module_globals, locals.Handle); + } - // add the imported module to the clr module, and copy the API functions - // and decorators into the main clr module. - Runtime.PyDict_SetItemString(clr_dict, "_extras", module); - foreach (PyObject key in locals.Keys()) + // add the imported module to the clr module, and copy the API functions + // and decorators into the main clr module. + Runtime.PyDict_SetItemString(clr_dict, "_extras", module); + using (var keys = locals.Keys()) + foreach (PyObject key in keys) + { + if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { - if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) - { - PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); - value.Dispose(); - } - key.Dispose(); + PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + value.Dispose(); } - } - finally - { - locals.Dispose(); + key.Dispose(); } } + finally + { + locals.Dispose(); + } + } + + static void OnDomainUnload(object _, EventArgs __) + { + Shutdown(); } /// @@ -228,11 +262,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) /// CPython interpreter process - this bootstraps the managed runtime /// when it is imported by the CLR extension module. /// -#if PYTHON3 public static IntPtr InitExt() -#elif PYTHON2 - public static void InitExt() -#endif { try { @@ -272,14 +302,10 @@ public static void InitExt() catch (PythonException e) { e.Restore(); -#if PYTHON3 return IntPtr.Zero; -#endif } -#if PYTHON3 return Python.Runtime.ImportHook.GetCLRModule(); -#endif } /// @@ -290,22 +316,102 @@ public static void InitExt() /// Python runtime can no longer be used in the current process /// after calling the Shutdown method. /// - public static void Shutdown() + /// The ShutdownMode to use when shutting down the Runtime + public static void Shutdown(ShutdownMode mode) { - if (initialized) + if (!initialized) { - Marshal.FreeHGlobal(_pythonHome); - _pythonHome = IntPtr.Zero; - Marshal.FreeHGlobal(_programName); - _programName = IntPtr.Zero; - Marshal.FreeHGlobal(_pythonPath); - _pythonPath = IntPtr.Zero; + return; + } + // If the shutdown handlers trigger a domain unload, + // don't call shutdown again. + AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + + PyScopeManager.Global.Clear(); + ExecuteShutdownHandlers(); + // Remember to shut down the runtime. + Runtime.Shutdown(mode); + PyObjectConversions.Reset(); + + initialized = false; + } + + /// + /// Shutdown Method + /// + /// + /// Shutdown and release resources held by the Python runtime. The + /// Python runtime can no longer be used in the current process + /// after calling the Shutdown method. + /// + public static void Shutdown() + { + Shutdown(Runtime.ShutdownMode); + } + + /// + /// Called when the engine is shut down. + /// + /// Shutdown handlers are run in reverse order they were added, so that + /// resources available when running a shutdown handler are the same as + /// what was available when it was added. + /// + public delegate void ShutdownHandler(); - Runtime.Shutdown(); - initialized = false; + static List ShutdownHandlers = new List(); + + /// + /// Add a function to be called when the engine is shut down. + /// + /// Shutdown handlers are executed in the opposite order they were + /// added, so that you can be sure that everything that was initialized + /// when you added the handler is still initialized when you need to shut + /// down. + /// + /// If the same shutdown handler is added several times, it will be run + /// several times. + /// + /// Don't add shutdown handlers while running a shutdown handler. + /// + public static void AddShutdownHandler(ShutdownHandler handler) + { + ShutdownHandlers.Add(handler); + } + + /// + /// Remove a shutdown handler. + /// + /// If the same shutdown handler is added several times, only the last + /// one is removed. + /// + /// Don't remove shutdown handlers while running a shutdown handler. + /// + public static void RemoveShutdownHandler(ShutdownHandler handler) + { + for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) + { + if (ShutdownHandlers[index] == handler) + { + ShutdownHandlers.RemoveAt(index); + break; + } } } + /// + /// Run all the shutdown handlers. + /// + /// They're run in opposite order they were added. + /// + static void ExecuteShutdownHandlers() + { + while(ShutdownHandlers.Count > 0) + { + var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; + ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); + handler(); + } + } /// /// AcquireLock Method @@ -385,7 +491,7 @@ public static void EndAllowThreads(IntPtr ts) public static PyObject ImportModule(string name) { IntPtr op = Runtime.PyImport_ImportModule(name); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(op); return new PyObject(op); } @@ -400,7 +506,7 @@ public static PyObject ImportModule(string name) public static PyObject ReloadModule(PyObject module) { IntPtr op = Runtime.PyImport_ReloadModule(module.Handle); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(op); return new PyObject(op); } @@ -414,13 +520,20 @@ public static PyObject ReloadModule(PyObject module) /// public static PyObject ModuleFromString(string name, string code) { - IntPtr c = Runtime.Py_CompileString(code, "none", (IntPtr)257); - Runtime.CheckExceptionOccurred(); + IntPtr c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); + PythonException.ThrowIfIsNull(c); IntPtr m = Runtime.PyImport_ExecCodeModule(name, c); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(m); return new PyObject(m); } + public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) + { + var flag = (int)mode; + IntPtr ptr = Runtime.Py_CompileString(code, filename, flag); + PythonException.ThrowIfIsNull(ptr); + return new PyObject(ptr); + } /// /// Eval Method @@ -445,12 +558,13 @@ public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals /// public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = null) { - PyObject result = RunString(code, globals, locals, RunFlagType.File); - if (result.obj != Runtime.PyNone) + using (PyObject result = RunString(code, globals, locals, RunFlagType.File)) { - throw new PythonException(); + if (result.obj != Runtime.PyNone) + { + throw new PythonException(); + } } - result.Dispose(); } @@ -481,37 +595,29 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, if (globals == IntPtr.Zero) { globals = Runtime.PyDict_New(); - Runtime.PyDict_SetItemString( - globals.Value, "__builtins__", + Runtime.PyDict_SetItem( + globals.Value, PyIdentifier.__builtins__, Runtime.PyEval_GetBuiltins() ); borrowedGlobals = false; } } - var borrowedLocals = true; if (locals == null) { - locals = Runtime.PyDict_New(); - borrowedLocals = false; + locals = globals; } try { - IntPtr result = Runtime.PyRun_String( - code, (IntPtr)flag, globals.Value, locals.Value + NewReference result = Runtime.PyRun_String( + code, flag, globals.Value, locals.Value ); - - Runtime.CheckExceptionOccurred(); - - return new PyObject(result); + PythonException.ThrowIfIsNull(result); + return result.MoveToPyObject(); } finally { - if (!borrowedLocals) - { - Runtime.XDecref(locals.Value); - } if (!borrowedGlobals) { Runtime.XDecref(globals.Value); @@ -520,7 +626,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, } } - public enum RunFlagType + public enum RunFlagType : int { Single = 256, File = 257, /* Py_file_input */ @@ -539,9 +645,22 @@ public static GILState GIL() return new GILState(); } + public static PyScope CreateScope() + { + var scope = PyScopeManager.Global.Create(); + return scope; + } + + public static PyScope CreateScope(string name) + { + var scope = PyScopeManager.Global.Create(name); + return scope; + } + public class GILState : IDisposable { - private IntPtr state; + private readonly IntPtr state; + private bool isDisposed; internal GILState() { @@ -550,8 +669,11 @@ internal GILState() public void Dispose() { + if (this.isDisposed) return; + PythonEngine.ReleaseLock(state); GC.SuppressFinalize(this); + this.isDisposed = true; } ~GILState() @@ -632,5 +754,37 @@ public static void SetArgv(IEnumerable argv) Runtime.CheckExceptionOccurred(); } } + + public static void With(PyObject obj, Action Body) + { + // Behavior described here: + // https://docs.python.org/2/reference/datamodel.html#with-statement-context-managers + + IntPtr type = Runtime.PyNone; + IntPtr val = Runtime.PyNone; + IntPtr traceBack = Runtime.PyNone; + PythonException ex = null; + + try + { + PyObject enterResult = obj.InvokeMethod("__enter__"); + + Body(enterResult); + } + catch (PythonException e) + { + ex = e; + type = ex.PyType.Coalesce(type); + val = ex.PyValue.Coalesce(val); + traceBack = ex.PyTB.Coalesce(traceBack); + } + + Runtime.XIncref(type); + Runtime.XIncref(val); + Runtime.XIncref(traceBack); + var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack)); + + if (ex != null && !exitResult.IsTrue()) throw ex; + } } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 4fe07f3cf..eff33d699 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,6 @@ using System; +using System.Runtime.CompilerServices; +using System.Text; namespace Python.Runtime { @@ -6,22 +8,21 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception + public class PythonException : System.Exception, IDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; private IntPtr _pyTB = IntPtr.Zero; private string _tb = ""; private string _message = ""; + private string _pythonTypeName = ""; private bool disposed = false; + private bool _finalized = false; public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); + Runtime.PyErr_Fetch(out _pyType, out _pyValue, out _pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; @@ -33,6 +34,8 @@ public PythonException() type = pyTypeName.ToString(); } + _pythonTypeName = type; + Runtime.XIncref(_pyValue); using (var pyValue = new PyObject(_pyValue)) { @@ -42,11 +45,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + using (PyObject tb_module = PythonEngine.ImportModule("traceback")) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + Runtime.XIncref(_pyTB); + using (var pyTB = new PyObject(_pyTB)) + { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } } PythonEngine.ReleaseLock(gs); @@ -57,7 +62,14 @@ public PythonException() ~PythonException() { - Dispose(); + if (_finalized || disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(ref _pyType); + Finalizer.Instance.AddFinalizedObject(ref _pyValue); + Finalizer.Instance.AddFinalizedObject(ref _pyTB); } /// @@ -125,9 +137,68 @@ public override string Message /// public override string StackTrace { - get { return _tb; } + get { return _tb + base.StackTrace; } } + /// + /// Python error type name. + /// + public string PythonTypeName + { + get { return _pythonTypeName; } + } + + /// + /// Formats this PythonException object into a message as would be printed + /// out via the Python console. See traceback.format_exception + /// + public string Format() + { + string res; + IntPtr gs = PythonEngine.AcquireLock(); + try + { + if (_pyTB != IntPtr.Zero && _pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + { + IntPtr tb = _pyTB; + IntPtr type = _pyType; + IntPtr value = _pyValue; + + Runtime.XIncref(type); + Runtime.XIncref(value); + Runtime.XIncref(tb); + Runtime.PyErr_NormalizeException(ref type, ref value, ref tb); + + using (PyObject pyType = new PyObject(type)) + using (PyObject pyValue = new PyObject(value)) + using (PyObject pyTB = new PyObject(tb)) + using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) + { + var buffer = new StringBuilder(); + var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); + foreach (PyObject val in values) + { + buffer.Append(val.ToString()); + } + res = buffer.ToString(); + } + } + else + { + res = StackTrace; + } + } + finally + { + PythonEngine.ReleaseLock(gs); + } + return res; + } + + public bool IsMatches(IntPtr exc) + { + return Runtime.PyErr_GivenExceptionMatches(PyType, exc) != 0; + } /// /// Dispose Method @@ -145,12 +216,23 @@ public void Dispose() if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(_pyType); - Runtime.XDecref(_pyValue); + if (_pyType != IntPtr.Zero) + { + Runtime.XDecref(_pyType); + _pyType= IntPtr.Zero; + } + + if (_pyValue != IntPtr.Zero) + { + Runtime.XDecref(_pyValue); + _pyValue = IntPtr.Zero; + } + // XXX Do we ever get TraceBack? // if (_pyTB != IntPtr.Zero) { Runtime.XDecref(_pyTB); + _pyTB = IntPtr.Zero; } PythonEngine.ReleaseLock(gs); } @@ -170,5 +252,32 @@ public static bool Matches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } + + [System.Diagnostics.DebuggerHidden] + public static void ThrowIfIsNull(IntPtr ob) + { + if (ob == IntPtr.Zero) + { + throw new PythonException(); + } + } + + [System.Diagnostics.DebuggerHidden] + internal static void ThrowIfIsNull(BorrowedReference reference) + { + if (reference.IsNull) + { + throw new PythonException(); + } + } + + [System.Diagnostics.DebuggerHidden] + public static void ThrowIfIsNotZero(int value) + { + if (value != 0) + { + throw new PythonException(); + } + } } } diff --git a/src/runtime/pytuple.cs b/src/runtime/pytuple.cs index 45f3d8350..530ced3d2 100644 --- a/src/runtime/pytuple.cs +++ b/src/runtime/pytuple.cs @@ -22,36 +22,35 @@ public PyTuple(IntPtr ptr) : base(ptr) { } - /// /// PyTuple Constructor /// /// - /// Copy constructor - obtain a PyTuple from a generic PyObject. An - /// ArgumentException will be thrown if the given object is not a - /// Python tuple object. + /// Creates a new PyTuple from an existing object reference. + /// The object reference is not checked for type-correctness. /// - public PyTuple(PyObject o) + internal PyTuple(BorrowedReference reference) : base(reference) { } + + private static IntPtr FromObject(PyObject o) { - if (!IsTupleType(o)) + if (o == null || !IsTupleType(o)) { throw new ArgumentException("object is not a tuple"); } Runtime.XIncref(o.obj); - obj = o.obj; + return o.obj; } - /// /// PyTuple Constructor /// /// - /// Creates a new empty PyTuple. + /// Copy constructor - obtain a PyTuple from a generic PyObject. An + /// ArgumentException will be thrown if the given object is not a + /// Python tuple object. /// - public PyTuple() + public PyTuple(PyObject o) : base(FromObject(o)) { - obj = Runtime.PyTuple_New(0); - Runtime.CheckExceptionOccurred(); } @@ -59,22 +58,42 @@ public PyTuple() /// PyTuple Constructor /// /// - /// Creates a new PyTuple from an array of PyObject instances. - /// - /// See caveats about PyTuple_SetItem: - /// https://www.coursehero.com/file/p4j2ogg/important-exceptions-to-this-rule-PyTupleSetItem-and-PyListSetItem-These/ + /// Creates a new empty PyTuple. /// - public PyTuple(PyObject[] items) + public PyTuple() : base(Runtime.PyTuple_New(0)) + { + PythonException.ThrowIfIsNull(obj); + } + + private static IntPtr FromArray(PyObject[] items) { int count = items.Length; - obj = Runtime.PyTuple_New(count); + IntPtr val = Runtime.PyTuple_New(count); for (var i = 0; i < count; i++) { IntPtr ptr = items[i].obj; Runtime.XIncref(ptr); - Runtime.PyTuple_SetItem(obj, i, ptr); - Runtime.CheckExceptionOccurred(); + int res = Runtime.PyTuple_SetItem(val, i, ptr); + if (res != 0) + { + Runtime.Py_DecRef(val); + throw new PythonException(); + } } + return val; + } + + /// + /// PyTuple Constructor + /// + /// + /// Creates a new PyTuple from an array of PyObject instances. + /// + /// See caveats about PyTuple_SetItem: + /// https://www.coursehero.com/file/p4j2ogg/important-exceptions-to-this-rule-PyTupleSetItem-and-PyListSetItem-These/ + /// + public PyTuple(PyObject[] items) : base(FromArray(items)) + { } @@ -101,7 +120,7 @@ public static bool IsTupleType(PyObject value) public static PyTuple AsTuple(PyObject value) { IntPtr op = Runtime.PySequence_Tuple(value.obj); - Runtime.CheckExceptionOccurred(); + PythonException.ThrowIfIsNull(op); return new PyTuple(op); } } diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 91f1f6b15..2254e7430 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.3.0" +__version__ = "3.0.0dev" class clrproperty(object): diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 346ea745f..63467c917 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1,78 +1,17 @@ +using System.Reflection.Emit; using System; +using System.Diagnostics.Contracts; using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Threading; +using System.Collections.Generic; +using Python.Runtime.Native; +using Python.Runtime.Platform; +using System.Linq; namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll)] - private static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } - /// /// Encapsulates the low-level Python C API. Note that it is /// the responsibility of the caller to have acquired the GIL @@ -80,52 +19,25 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) /// public class Runtime { -#if UCS4 - public const int UCS = 4; - - /// - /// EntryPoint to be used in DllImport to map to correct Unicode - /// methods prior to PEP393. Only used for PY27. - /// - private const string PyUnicodeEntryPoint = "PyUnicodeUCS4_"; -#elif UCS2 - public const int UCS = 2; - - /// - /// EntryPoint to be used in DllImport to map to correct Unicode - /// methods prior to PEP393. Only used for PY27. - /// - private const string PyUnicodeEntryPoint = "PyUnicodeUCS2_"; + public static int UCS => _UCS; + internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; + +#if PYTHON36 + const string _minor = "6"; +#elif PYTHON37 + const string _minor = "7"; +#elif PYTHON38 + const string _minor = "8"; +#elif PYTHON39 + const string _minor = "9"; #else -#error You must define either UCS2 or UCS4! +#error You must define one of PYTHON36 to PYTHON39 #endif -#if PYTHON27 - public const string pyversion = "2.7"; - public const string pyver = "27"; -#elif PYTHON33 - public const string pyversion = "3.3"; - public const string pyver = "33"; -#elif PYTHON34 - public const string pyversion = "3.4"; - public const string pyver = "34"; -#elif PYTHON35 - public const string pyversion = "3.5"; - public const string pyver = "35"; -#elif PYTHON36 - public const string pyversion = "3.6"; - public const string pyver = "36"; -#elif PYTHON37 // TODO: Add `interop37.cs` after PY37 is released - public const string pyversion = "3.7"; - public const string pyver = "37"; +#if WINDOWS + internal const string dllBase = "python3" + _minor; #else -#error You must define one of PYTHON33 to PYTHON37 or PYTHON27 -#endif - -#if MONO_LINUX || MONO_OSX // Linux/macOS use dotted version string - internal const string dllBase = "python" + pyversion; -#else // Windows - internal const string dllBase = "python" + pyver; + internal const string dllBase = "python3." + _minor; #endif #if PYTHON_WITH_PYDEBUG @@ -139,178 +51,372 @@ public class Runtime internal const string dllWithPyMalloc = ""; #endif -#if PYTHON_WITHOUT_ENABLE_SHARED - public const string PythonDll = "__Internal"; + // C# compiler copies constants to the assemblies that references this library. + // We needs to replace all public constants to static readonly fields to allow + // binary substitution of different Python.Runtime.dll builds in a target application. + + public static readonly string PythonDLL = _PythonDll; + +#if PYTHON_WITHOUT_ENABLE_SHARED && !NETSTANDARD + internal const string _PythonDll = "__Internal"; #else - public const string PythonDll = dllBase + dllWithPyDebug + dllWithPyMalloc; + internal const string _PythonDll = dllBase + dllWithPyDebug + dllWithPyMalloc; #endif - public static readonly int pyversionnumber = Convert.ToInt32(pyver); - // set to true when python is finalizing internal static object IsFinalizingLock = new object(); internal static bool IsFinalizing; - internal static bool Is32Bit = IntPtr.Size == 4; - internal static bool IsPython2 = pyversionnumber < 30; - internal static bool IsPython3 = pyversionnumber >= 30; + private static bool _isInitialized = false; + + internal static readonly bool Is32Bit = IntPtr.Size == 4; + + // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; + + internal static Version InteropVersion { get; } + = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; + + public static int MainManagedThreadId { get; private set; } /// /// Encoding to use to convert Unicode to/from Managed to Native /// - internal static readonly Encoding PyEncoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + + public static ShutdownMode ShutdownMode { get; internal set; } + private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); + + internal static Version PyVersion + { + get + { + using (var versionTuple = new PyTuple(PySys_GetObject("version_info"))) + { + var major = versionTuple[0].As(); + var minor = versionTuple[1].As(); + var micro = versionTuple[2].As(); + return new Version(major, minor, micro); + } + } + } + /// /// Initialize the runtime... /// - internal static void Initialize() + /// Always call this method from the Main thread. After the + /// first call to this method, the main thread has acquired the GIL. + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - if (Py_IsInitialized() == 0) + if (_isInitialized) { - Py_Initialize(); + return; } + _isInitialized = true; - if (PyEval_ThreadsInitialized() == 0) + if (mode == ShutdownMode.Default) { - PyEval_InitThreads(); + mode = GetDefaultShutdownMode(); } + ShutdownMode = mode; - IntPtr op; - IntPtr dict; - if (IsPython3) + if (Py_IsInitialized() == 0) { - op = PyImport_ImportModule("builtins"); - dict = PyObject_GetAttrString(op, "__dict__"); + Py_InitializeEx(initSigs ? 1 : 0); + if (PyEval_ThreadsInitialized() == 0) + { + PyEval_InitThreads(); + } + // XXX: Reload mode may reduct to Soft mode, + // so even on Reload mode it still needs to save the RuntimeState + if (mode == ShutdownMode.Soft || mode == ShutdownMode.Reload) + { + RuntimeState.Save(); + } } - else // Python2 + else { - dict = PyImport_GetModuleDict(); - op = PyDict_GetItemString(dict, "__builtin__"); + // If we're coming back from a domain reload or a soft shutdown, + // we have previously released the thread state. Restore the main + // thread state here. + PyGILState_Ensure(); } - PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); - PyBaseObjectType = PyObject_GetAttrString(op, "object"); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; - PyModuleType = PyObject_Type(op); - PyNone = PyObject_GetAttrString(op, "None"); - PyTrue = PyObject_GetAttrString(op, "True"); - PyFalse = PyObject_GetAttrString(op, "False"); + IsFinalizing = false; + InternString.Initialize(); - PyBoolType = PyObject_Type(PyTrue); - PyNoneType = PyObject_Type(PyNone); - PyTypeType = PyObject_Type(PyNoneType); + InitPyMembers(); - op = PyObject_GetAttrString(dict, "keys"); - PyMethodType = PyObject_Type(op); - XDecref(op); + ABI.Initialize(PyVersion, + pyType: new BorrowedReference(PyTypeType)); - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - PyWrapperDescriptorType = PyObject_Type(op); - XDecref(op); + GenericUtil.Reset(); + PyScopeManager.Reset(); + ClassManager.Reset(); + ClassDerivedObject.Reset(); + TypeManager.Initialize(); -#if PYTHON3 - XDecref(dict); -#endif + // Initialize modules that depend on the runtime class. + AssemblyManager.Initialize(); + OperatorMethod.Initialize(); + if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) + { + RuntimeData.RestoreRuntimeData(); + } + else + { + PyCLRMetaType = MetaType.Initialize(); // Steal a reference + ImportHook.Initialize(); + } + Exceptions.Initialize(); + + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + IntPtr path = PySys_GetObject("path").DangerousGetAddress(); + IntPtr item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item) == 0) + { + PyList_Append(new BorrowedReference(path), item); + } + XDecref(item); + AssemblyManager.UpdatePath(); + } + + private static void InitPyMembers() + { + IntPtr op; + { + var builtins = GetBuiltins(); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented"), + () => PyNotImplemented = IntPtr.Zero); + + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object"), + () => PyBaseObjectType = IntPtr.Zero); + + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None"), + () => PyNone = IntPtr.Zero); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True"), + () => PyTrue = IntPtr.Zero); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), + () => PyFalse = IntPtr.Zero); + + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + () => PyBoolType = IntPtr.Zero); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + () => PyNoneType = IntPtr.Zero); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + () => PyTypeType = IntPtr.Zero); + + op = PyObject_GetAttrString(builtins, "len"); + SetPyMember(ref PyMethodType, PyObject_Type(op), + () => PyMethodType = IntPtr.Zero); + XDecref(op); + + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + op = PyObject_GetAttr(PyBaseObjectType, PyIdentifier.__init__); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + () => PyWrapperDescriptorType = IntPtr.Zero); + XDecref(op); + + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super"), + () => PySuper_Type = IntPtr.Zero); + + XDecref(builtins); + } op = PyString_FromString("string"); - PyStringType = PyObject_Type(op); + SetPyMember(ref PyStringType, PyObject_Type(op), + () => PyStringType = IntPtr.Zero); XDecref(op); op = PyUnicode_FromString("unicode"); - PyUnicodeType = PyObject_Type(op); + SetPyMember(ref PyUnicodeType, PyObject_Type(op), + () => PyUnicodeType = IntPtr.Zero); XDecref(op); -#if PYTHON3 op = PyBytes_FromString("bytes"); - PyBytesType = PyObject_Type(op); + SetPyMember(ref PyBytesType, PyObject_Type(op), + () => PyBytesType = IntPtr.Zero); XDecref(op); -#endif op = PyTuple_New(0); - PyTupleType = PyObject_Type(op); + SetPyMember(ref PyTupleType, PyObject_Type(op), + () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - PyListType = PyObject_Type(op); + SetPyMember(ref PyListType, PyObject_Type(op), + () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - PyDictType = PyObject_Type(op); + SetPyMember(ref PyDictType, PyObject_Type(op), + () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - PyIntType = PyObject_Type(op); + SetPyMember(ref PyIntType, PyObject_Type(op), + () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - PyLongType = PyObject_Type(op); + SetPyMember(ref PyLongType, PyObject_Type(op), + () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - PyFloatType = PyObject_Type(op); + SetPyMember(ref PyFloatType, PyObject_Type(op), + () => PyFloatType = IntPtr.Zero); XDecref(op); -#if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; -#elif PYTHON2 - IntPtr s = PyString_FromString("_temp"); - IntPtr d = PyDict_New(); - IntPtr c = PyClass_New(IntPtr.Zero, d, s); - PyClassType = PyObject_Type(c); + Error = new IntPtr(-1); - IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - PyInstanceType = PyObject_Type(i); + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + IntPtr sys = PyImport_ImportModule("sys"); + PyModuleType = PyObject_Type(sys); + XDecref(sys); + } + } - XDecref(s); - XDecref(i); - XDecref(c); - XDecref(d); -#endif + private static IntPtr Get_PyObject_NextNotImplemented() + { + IntPtr pyType = SlotHelper.CreateObjectType(); + IntPtr iternext = Marshal.ReadIntPtr(pyType, TypeOffset.tp_iternext); + Runtime.XDecref(pyType); + return iternext; + } - Error = new IntPtr(-1); + /// + /// Tries to downgrade the shutdown mode, if possible. + /// The only possibles downgrades are: + /// Soft -> Normal + /// Reload -> Soft + /// Reload -> Normal + /// + /// The desired shutdown mode + /// The `mode` parameter if the downgrade is supported, the ShutdownMode + /// set at initialization otherwise. + static ShutdownMode TryDowngradeShutdown(ShutdownMode mode) + { + if ( + mode == Runtime.ShutdownMode + || mode == ShutdownMode.Normal + || (mode == ShutdownMode.Soft && Runtime.ShutdownMode == ShutdownMode.Reload) + ) + { + return mode; + } + else // we can't downgrade + { + return Runtime.ShutdownMode; + } + } -#if PYTHON3 - IntPtr dllLocal = IntPtr.Zero; - if (PythonDll != "__Internal") + internal static void Shutdown(ShutdownMode mode) + { + if (Py_IsInitialized() == 0 || !_isInitialized) + { + return; + } + _isInitialized = false; + + // If the shutdown mode specified is not the the same as the one specified + // during Initialization, we need to validate it; we can only downgrade, + // not upgrade the shutdown mode. + mode = TryDowngradeShutdown(mode); + + var state = PyGILState_Ensure(); + + if (mode == ShutdownMode.Soft) { - NativeMethods.LoadLibrary(PythonDll); + RunExitFuncs(); } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); -#if !(MONO_LINUX || MONO_OSX) - if (dllLocal != IntPtr.Zero) + if (mode == ShutdownMode.Reload) { - NativeMethods.FreeLibrary(dllLocal); + RuntimeData.Stash(); } -#endif -#endif + AssemblyManager.Shutdown(); + OperatorMethod.Shutdown(); + ImportHook.Shutdown(); - // Initialize modules that depend on the runtime class. - AssemblyManager.Initialize(); - PyCLRMetaType = MetaType.Initialize(); - Exceptions.Initialize(); - ImportHook.Initialize(); + ClearClrModules(); + RemoveClrRootModule(); - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - IntPtr path = PySys_GetObject("path"); - IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); - XDecref(item); - AssemblyManager.UpdatePath(); + MoveClrInstancesOnwershipToPython(); + ClassManager.DisposePythonWrappersForClrTypes(); + TypeManager.RemoveTypes(); + + MetaType.Release(); + PyCLRMetaType = IntPtr.Zero; + + Exceptions.Shutdown(); + Finalizer.Shutdown(); + InternString.Shutdown(); + + if (mode != ShutdownMode.Normal) + { + PyGC_Collect(); + if (mode == ShutdownMode.Soft) + { + RuntimeState.Restore(); + } + ResetPyMembers(); + GC.Collect(); + try + { + GC.WaitForFullGCComplete(); + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + } + GC.WaitForPendingFinalizers(); + PyGILState_Release(state); + // Then release the GIL for good, if there is somehting to release + // Use the unchecked version as the checked version calls `abort()` + // if the current state is NULL. + if (_PyThreadState_UncheckedGet() != IntPtr.Zero) + { + PyEval_SaveThread(); + } + + } + else + { + ResetPyMembers(); + Py_Finalize(); + } } internal static void Shutdown() { - AssemblyManager.Shutdown(); - Exceptions.Shutdown(); - ImportHook.Shutdown(); - Py_Finalize(); + var mode = ShutdownMode; + Shutdown(mode); + } + + internal static ShutdownMode GetDefaultShutdownMode() + { + string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); + if (modeEvn == null) + { + return ShutdownMode.Normal; + } + ShutdownMode mode; + if (Enum.TryParse(modeEvn, true, out mode)) + { + return mode; + } + return ShutdownMode.Normal; } // called *without* the GIL acquired by clr._AtExit @@ -323,14 +429,124 @@ internal static int AtExit() return 0; } - internal static IntPtr Py_single_input = (IntPtr)256; - internal static IntPtr Py_file_input = (IntPtr)257; - internal static IntPtr Py_eval_input = (IntPtr)258; + private static void RunExitFuncs() + { + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + if (!e.IsMatches(Exceptions.ImportError)) + { + throw; + } + e.Dispose(); + // The runtime may not provided `atexit` module. + return; + } + using (atexit) + { + try + { + atexit.InvokeMethod("_run_exitfuncs").Dispose(); + } + catch (PythonException e) + { + Console.Error.WriteLine(e); + e.Dispose(); + } + } + } + + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) + { + // XXX: For current usages, value should not be null. + PythonException.ThrowIfIsNull(value); + obj = value; + _pyRefs.Add(value, onRelease); + } + + private static void ResetPyMembers() + { + _pyRefs.Release(); + } + + private static void ClearClrModules() + { + var modules = PyImport_GetModuleDict(); + var items = PyDict_Items(modules); + long length = PyList_Size(items); + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + if (ManagedType.IsManagedType(module)) + { + PyDict_DelItem(modules, name); + } + } + items.Dispose(); + } + + private static void RemoveClrRootModule() + { + var modules = PyImport_GetModuleDict(); + PyDictTryDelItem(modules, "clr"); + PyDictTryDelItem(modules, "clr._extra"); + } + + private static void PyDictTryDelItem(IntPtr dict, string key) + { + if (PyDict_DelItemString(dict, key) == 0) + { + return; + } + if (!PythonException.Matches(Exceptions.KeyError)) + { + throw new PythonException(); + } + PyErr_Clear(); + } + + private static void MoveClrInstancesOnwershipToPython() + { + var objs = ManagedType.GetManagedObjects(); + var copyObjs = objs.ToArray(); + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; + if (!objs.ContainsKey(obj)) + { + System.Diagnostics.Debug.Assert(obj.gcHandle == default); + continue; + } + if (entry.Value == ManagedType.TrackTypes.Extension) + { + obj.CallTypeClear(); + // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), + // thus just be safe to give it back to GC chain. + if (!_PyObject_GC_IS_TRACKED(obj.pyHandle)) + { + PyObject_GC_Track(obj.pyHandle); + } + } + if (obj.gcHandle.IsAllocated) + { + obj.gcHandle.Free(); + } + obj.gcHandle = default; + } + ManagedType.ClearTrackedObjects(); + } internal static IntPtr PyBaseObjectType; internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -347,10 +563,10 @@ internal static int AtExit() internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; -#if PYTHON3 + internal static IntPtr Py_NoSiteFlag; + internal static IntPtr PyBytesType; internal static IntPtr _PyObject_NextNotImplemented; -#endif internal static IntPtr PyNotImplemented; internal const int Py_LT = 0; @@ -365,6 +581,16 @@ internal static int AtExit() internal static IntPtr PyNone; internal static IntPtr Error; + public static PyObject None + { + get + { + var none = Runtime.PyNone; + Runtime.XIncref(none); + return new PyObject(none); + } + } + /// /// Check if any Python Exceptions occurred. /// If any exist throw new PythonException. @@ -374,38 +600,15 @@ internal static int AtExit() /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != 0) + if (PyErr_Occurred() != IntPtr.Zero) { throw new PythonException(); } } - internal static IntPtr GetBoundArgTuple(IntPtr obj, IntPtr args) - { - if (PyObject_TYPE(args) != PyTupleType) - { - Exceptions.SetError(Exceptions.TypeError, "tuple expected"); - return IntPtr.Zero; - } - int size = PyTuple_Size(args); - IntPtr items = PyTuple_New(size + 1); - PyTuple_SetItem(items, 0, obj); - XIncref(obj); - - for (var i = 0; i < size; i++) - { - IntPtr item = PyTuple_GetItem(args, i); - XIncref(item); - PyTuple_SetItem(items, i + 1, item); - } - - return items; - } - - internal static IntPtr ExtendTuple(IntPtr t, params IntPtr[] args) { - int size = PyTuple_Size(t); + var size = PyTuple_Size(t); int add = args.Length; IntPtr item; @@ -448,7 +651,7 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) free = true; } - int n = PyTuple_Size(args); + var n = PyTuple_Size(args); var types = new Type[n]; Type t = null; @@ -463,7 +666,8 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) if (mt is ClassBase) { - t = ((ClassBase)mt).type; + MaybeType _type = ((ClassBase)mt).type; + t = _type.Valid ? _type.Value : null; } else if (mt is CLRObject) { @@ -499,7 +703,7 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) /// internal static unsafe void XIncref(IntPtr op) { -#if PYTHON_WITH_PYDEBUG +#if !CUSTOM_INCDEC_REF Py_IncRef(op); return; #else @@ -518,9 +722,18 @@ internal static unsafe void XIncref(IntPtr op) #endif } + /// + /// Increase Python's ref counter for the given object, and get the object back. + /// + internal static IntPtr SelfIncRef(IntPtr op) + { + XIncref(op); + return op; + } + internal static unsafe void XDecref(IntPtr op) { -#if PYTHON_WITH_PYDEBUG +#if !CUSTOM_INCDEC_REF Py_DecRef(op); return; #else @@ -549,15 +762,20 @@ internal static unsafe void XDecref(IntPtr op) { return; } - NativeCall.Impl.Void_Call_1(new IntPtr(f), op); + NativeCall.Void_Call_1(new IntPtr(f), op); } } #endif } + [Pure] internal static unsafe long Refcount(IntPtr op) { +#if PYTHON_WITH_PYDEBUG + var p = (void*)(op + TypeOffset.ob_refcnt); +#else var p = (void*)op; +#endif if ((void*)0 == p) { return 0; @@ -570,7 +788,7 @@ internal static unsafe long Refcount(IntPtr op) /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_IncRef(IntPtr ob); /// @@ -578,157 +796,175 @@ internal static unsafe long Refcount(IntPtr op) /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_DecRef(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_Initialize(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void Py_InitializeEx(int initsigs); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_IsInitialized(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_Finalize(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_NewInterpreter(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_EndInterpreter(IntPtr threadState); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThreadState_New(IntPtr istate); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThreadState_Get(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr _PyThreadState_UncheckedGet(); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThread_get_key_value(IntPtr key); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyThread_get_thread_ident(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyThread_set_key_value(IntPtr key, IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThreadState_Swap(IntPtr key); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyGILState_Ensure(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyGILState_Release(IntPtr gs); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyGILState_GetThisThreadState(); -#if PYTHON3 - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] public static extern int Py_Main( int argc, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv ); -#elif PYTHON2 - [DllImport(PythonDll)] - public static extern int Py_Main(int argc, string[] argv); -#endif - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_InitThreads(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyEval_ThreadsInitialized(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_AcquireLock(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_ReleaseLock(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_AcquireThread(IntPtr tstate); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_ReleaseThread(IntPtr tstate); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_SaveThread(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_RestoreThread(IntPtr tstate); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_GetBuiltins(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_GetGlobals(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_GetLocals(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetProgramName(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_SetProgramName(IntPtr name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetPythonHome(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_SetPythonHome(IntPtr home); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetPath(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_SetPath(IntPtr home); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetVersion(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetPlatform(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetCopyright(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetCompiler(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr Py_GetBuildInfo(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyRun_SimpleString(string code); - [DllImport(PythonDll)] - internal static extern IntPtr PyRun_String(string code, IntPtr st, IntPtr globals, IntPtr locals); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, RunFlagType st, IntPtr globals, IntPtr locals); - [DllImport(PythonDll)] - internal static extern IntPtr Py_CompileString(string code, string file, IntPtr tok); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); - [DllImport(PythonDll)] - internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. + /// + internal static IntPtr Py_CompileString(string str, string file, int start) + { + return Py_CompileStringFlags(str, file, start, IntPtr.Zero); + } - [DllImport(PythonDll)] - internal static extern IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod); + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringExFlags() below, with optimize set to -1. + /// + internal static IntPtr Py_CompileStringFlags(string str, string file, int start, IntPtr flags) + { + return Py_CompileStringExFlags(str, file, start, flags, -1); + } - [DllImport(PythonDll)] - internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); + /// + /// Return value: New reference. + /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). + /// + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize); - [DllImport(PythonDll)] - internal static extern IntPtr PyClass_New(IntPtr bases, IntPtr dict, IntPtr name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); - [DllImport(PythonDll)] - internal static extern IntPtr PyInstance_New(IntPtr cls, IntPtr args, IntPtr kw); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod); - [DllImport(PythonDll)] - internal static extern IntPtr PyInstance_NewRaw(IntPtr cls, IntPtr dict); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_New(IntPtr func, IntPtr self, IntPtr cls); @@ -737,6 +973,7 @@ public static extern int Py_Main( //==================================================================== /// + /// Return value: Borrowed reference. /// A macro-like method to get the type of a Python object. This is /// designed to be lean and mean in IL & avoid managed <-> unmanaged /// transitions. Note that this does not incref the type object. @@ -757,6 +994,8 @@ internal static unsafe IntPtr PyObject_TYPE(IntPtr op) ? new IntPtr((void*)(*((uint*)p + n))) : new IntPtr((void*)(*((ulong*)p + n))); } + internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op) + => new BorrowedReference(PyObject_TYPE(op.DangerousGetAddress())); /// /// Managed version of the standard Python C API PyObject_Type call. @@ -777,44 +1016,53 @@ internal static string PyObject_GetTypeName(IntPtr op) return Marshal.PtrToStringAnsi(ppName); } - [DllImport(PythonDll)] - internal static extern int PyObject_HasAttrString(IntPtr pointer, string name); + /// + /// Test whether the Python object is an iterable. + /// + internal static bool PyObject_IsIterable(IntPtr pointer) + { + var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + IntPtr tp_iter = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iter); + return tp_iter != IntPtr.Zero; + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_HasAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name); - [DllImport(PythonDll)] - internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, string name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name); - [DllImport(PythonDll)] - internal static extern int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_SetAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name, IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_HasAttr(IntPtr pointer, IntPtr name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_SetAttr(IntPtr pointer, IntPtr name, IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GetItem(IntPtr pointer, IntPtr key); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_SetItem(IntPtr pointer, IntPtr key, IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_DelItem(IntPtr pointer, IntPtr key); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GetIter(IntPtr op); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args); -#if PYTHON3 - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid); internal static int PyObject_Compare(IntPtr value1, IntPtr value2) @@ -841,71 +1089,101 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) Exceptions.SetError(Exceptions.SystemError, "Error comparing objects"); return -1; } -#elif PYTHON2 - [DllImport(PythonDll)] - internal static extern int PyObject_Compare(IntPtr value1, IntPtr value2); -#endif - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_IsInstance(IntPtr ob, IntPtr type); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_IsSubclass(IntPtr ob, IntPtr type); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyCallable_Check(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_IsTrue(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_Not(IntPtr pointer); - [DllImport(PythonDll)] - internal static extern int PyObject_Size(IntPtr pointer); + internal static long PyObject_Size(IntPtr pointer) + { + return (long)_PyObject_Size(pointer); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] + private static extern IntPtr _PyObject_Size(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Hash(IntPtr op); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Repr(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Str(IntPtr pointer); -#if PYTHON3 - [DllImport(PythonDll, EntryPoint = "PyObject_Str")] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyObject_Str")] internal static extern IntPtr PyObject_Unicode(IntPtr pointer); -#elif PYTHON2 - [DllImport(PythonDll)] - internal static extern IntPtr PyObject_Unicode(IntPtr pointer); -#endif - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Dir(IntPtr pointer); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _Py_NewReference(IntPtr ob); +#endif + + //==================================================================== + // Python buffer API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyBuffer_Release(ref Py_buffer view); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + internal static extern IntPtr PyBuffer_SizeFromFormat([MarshalAs(UnmanagedType.LPStr)] string format); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBuffer_IsContiguous(ref Py_buffer view, char order); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags); //==================================================================== // Python number API //==================================================================== -#if PYTHON3 - [DllImport(PythonDll, EntryPoint = "PyNumber_Long")] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyNumber_Long")] internal static extern IntPtr PyNumber_Int(IntPtr ob); -#elif PYTHON2 - [DllImport(PythonDll)] - internal static extern IntPtr PyNumber_Int(IntPtr ob); -#endif - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Long(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Float(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PyNumber_Check(IntPtr ob); + internal static bool PyInt_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, new BorrowedReference(PyIntType)); internal static bool PyInt_Check(IntPtr ob) { return PyObject_TypeCheck(ob, PyIntType); @@ -928,65 +1206,79 @@ internal static IntPtr PyInt_FromInt64(long value) return PyInt_FromLong(v); } -#if PYTHON3 - [DllImport(PythonDll, EntryPoint = "PyLong_FromLong")] - private static extern IntPtr PyInt_FromLong(IntPtr value); - - [DllImport(PythonDll, EntryPoint = "PyLong_AsLong")] - internal static extern int PyInt_AsLong(IntPtr value); - - [DllImport(PythonDll, EntryPoint = "PyLong_FromString")] - internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); - - [DllImport(PythonDll, EntryPoint = "PyLong_GetMax")] - internal static extern int PyInt_GetMax(); -#elif PYTHON2 - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromLong")] private static extern IntPtr PyInt_FromLong(IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsLong")] internal static extern int PyInt_AsLong(IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromString")] internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); - [DllImport(PythonDll)] - internal static extern int PyInt_GetMax(); -#endif - internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyLongType; } - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromLong(long value); - [DllImport(PythonDll)] - internal static extern IntPtr PyLong_FromUnsignedLong(uint value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong32(uint value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong64(ulong value); + + internal static IntPtr PyLong_FromUnsignedLong(object value) + { + if (Is32Bit || IsWindows) + return PyLong_FromUnsignedLong32(Convert.ToUInt32(value)); + else + return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); + } - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromDouble(double value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromLongLong(long value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromUnsignedLongLong(ulong value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromString(string value, IntPtr end, int radix); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyLong_AsLong(IntPtr value); - [DllImport(PythonDll)] - internal static extern uint PyLong_AsUnsignedLong(IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern uint PyLong_AsUnsignedLong32(IntPtr value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern ulong PyLong_AsUnsignedLong64(IntPtr value); + + internal static object PyLong_AsUnsignedLong(IntPtr value) + { + if (Is32Bit || IsWindows) + return PyLong_AsUnsignedLong32(value); + else + return PyLong_AsUnsignedLong64(value); + } - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern long PyLong_AsLongLong(BorrowedReference value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern long PyLong_AsLongLong(IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern ulong PyLong_AsUnsignedLongLong(IntPtr value); internal static bool PyFloat_Check(IntPtr ob) @@ -994,88 +1286,101 @@ internal static bool PyFloat_Check(IntPtr ob) return PyObject_TYPE(ob) == PyFloatType; } - [DllImport(PythonDll)] + /// + /// Return value: New reference. + /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_FromVoidPtr(IntPtr p); + + /// + /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_AsVoidPtr(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyFloat_FromDouble(double value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyFloat_FromString(IntPtr value, IntPtr junk); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern double PyFloat_AsDouble(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Add(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Subtract(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] - internal static extern IntPtr PyNumber_Divide(IntPtr o1, IntPtr o2); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_And(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Xor(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Or(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Lshift(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Rshift(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Power(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Remainder(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] - internal static extern IntPtr PyNumber_InPlaceDivide(IntPtr o1, IntPtr o2); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceXor(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceOr(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlacePower(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Negative(IntPtr o1); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Positive(IntPtr o1); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Invert(IntPtr o1); @@ -1083,49 +1388,94 @@ internal static bool PyFloat_Check(IntPtr ob) // Python sequence API //==================================================================== - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PySequence_Check(IntPtr pointer); - [DllImport(PythonDll)] - internal static extern IntPtr PySequence_GetItem(IntPtr pointer, int index); + internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) + { + return PySequence_GetItem(pointer, new IntPtr(index)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); + + internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) + { + return PySequence_SetItem(pointer, new IntPtr(index), value); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + internal static int PySequence_DelItem(IntPtr pointer, long index) + { + return PySequence_DelItem(pointer, new IntPtr(index)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); + + internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) + { + return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); + } - [DllImport(PythonDll)] - internal static extern int PySequence_SetItem(IntPtr pointer, int index, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); + + internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) + { + return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); + } - [DllImport(PythonDll)] - internal static extern int PySequence_DelItem(IntPtr pointer, int index); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); - [DllImport(PythonDll)] - internal static extern IntPtr PySequence_GetSlice(IntPtr pointer, int i1, int i2); + internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) + { + return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); + } - [DllImport(PythonDll)] - internal static extern int PySequence_SetSlice(IntPtr pointer, int i1, int i2, IntPtr v); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - [DllImport(PythonDll)] - internal static extern int PySequence_DelSlice(IntPtr pointer, int i1, int i2); + internal static long PySequence_Size(IntPtr pointer) + { + return (long)_PySequence_Size(pointer); + } - [DllImport(PythonDll)] - internal static extern int PySequence_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] + private static extern IntPtr _PySequence_Size(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); - [DllImport(PythonDll)] - internal static extern IntPtr PySequence_Repeat(IntPtr pointer, int count); + internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) + { + return PySequence_Repeat(pointer, new IntPtr(count)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); - [DllImport(PythonDll)] - internal static extern int PySequence_Count(IntPtr pointer, IntPtr value); + internal static long PySequence_Count(IntPtr pointer, IntPtr value) + { + return (long)_PySequence_Count(pointer, value); + } - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] + private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Tuple(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_List(IntPtr pointer); @@ -1146,100 +1496,95 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { - return PyString_FromStringAndSize(value, value.Length); + return PyUnicode_FromKindAndData(_UCS, value, value.Length); } -#if PYTHON3 - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyBytes_FromString(string op); - [DllImport(PythonDll)] - internal static extern int PyBytes_Size(IntPtr op); + internal static long PyBytes_Size(IntPtr op) + { + return (long)_PyBytes_Size(op); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] + private static extern IntPtr _PyBytes_Size(IntPtr op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { return ob + BytesOffset.ob_sval; } - [DllImport(PythonDll, EntryPoint = "PyUnicode_FromStringAndSize")] - internal static extern IntPtr PyString_FromStringAndSize( - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value, - int size - ); - [DllImport(PythonDll)] - internal static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, int size); -#elif PYTHON2 - [DllImport(PythonDll)] - internal static extern IntPtr PyString_FromStringAndSize(string value, int size); + internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) + { + return PyUnicode_FromStringAndSize(value, new IntPtr(size)); + } - [DllImport(PythonDll)] - internal static extern IntPtr PyString_AsString(IntPtr op); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); - [DllImport(PythonDll)] - internal static extern int PyString_Size(IntPtr pointer); -#endif + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); internal static bool PyUnicode_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyUnicodeType; } -#if PYTHON3 - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromObject(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - [DllImport(PythonDll)] - internal static extern IntPtr PyUnicode_FromKindAndData( + internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) + { + return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyUnicode_FromKindAndData( int kind, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - int size + IntPtr size ); - internal static IntPtr PyUnicode_FromUnicode(string s, int size) + internal static IntPtr PyUnicode_FromUnicode(string s, long size) { - return PyUnicode_FromKindAndData(UCS, s, size); + return PyUnicode_FromKindAndData(_UCS, s, size); } - [DllImport(PythonDll)] - internal static extern int PyUnicode_GetSize(IntPtr ob); - - [DllImport(PythonDll)] - internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - - [DllImport(PythonDll)] - internal static extern IntPtr PyUnicode_FromOrdinal(int c); -#elif PYTHON2 - [DllImport(PythonDll, EntryPoint = PyUnicodeEntryPoint + "FromObject")] - internal static extern IntPtr PyUnicode_FromObject(IntPtr ob); - - [DllImport(PythonDll, EntryPoint = PyUnicodeEntryPoint + "FromEncodedObject")] - internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyUnicode_GetMax(); - [DllImport(PythonDll, EntryPoint = PyUnicodeEntryPoint + "FromUnicode")] - internal static extern IntPtr PyUnicode_FromUnicode( - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - int size - ); + internal static long PyUnicode_GetSize(IntPtr ob) + { + return (long)_PyUnicode_GetSize(ob); + } - [DllImport(PythonDll, EntryPoint = PyUnicodeEntryPoint + "GetSize")] - internal static extern int PyUnicode_GetSize(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] + private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); - [DllImport(PythonDll, EntryPoint = PyUnicodeEntryPoint + "AsUnicode")] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - [DllImport(PythonDll, EntryPoint = PyUnicodeEntryPoint + "FromOrdinal")] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromOrdinal(int c); -#endif internal static IntPtr PyUnicode_FromString(string s) { return PyUnicode_FromUnicode(s, s.Length); } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyUnicode_InternFromString(string s); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyUnicode_Compare(IntPtr left, IntPtr right); + + internal static string GetManagedString(in BorrowedReference borrowedReference) + => GetManagedString(borrowedReference.DangerousGetAddress()); /// /// Function to access the internal PyUnicode/PyString object and /// convert it to a managed string with the correct encoding. @@ -1257,19 +1602,12 @@ internal static string GetManagedString(IntPtr op) { IntPtr type = PyObject_TYPE(op); -#if PYTHON2 // Python 3 strings are all Unicode - if (type == PyStringType) - { - return Marshal.PtrToStringAnsi(PyString_AsString(op), PyString_Size(op)); - } -#endif - if (type == PyUnicodeType) { IntPtr p = PyUnicode_AsUnicode(op); - int length = PyUnicode_GetSize(op); + int length = (int)PyUnicode_GetSize(op); - int size = length * UCS; + int size = length * _UCS; var buffer = new byte[size]; Marshal.Copy(p, buffer, 0, size); return PyEncoding.GetString(buffer, 0, size); @@ -1288,54 +1626,92 @@ internal static bool PyDict_Check(IntPtr ob) return PyObject_TYPE(ob) == PyDictType; } - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_New(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDictProxy_New(IntPtr dict); - [DllImport(PythonDll)] + /// + /// Return value: Borrowed reference. + /// Return NULL if the key key is not present, but without setting an exception. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key); - [DllImport(PythonDll)] - internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, string key); + /// + /// Return value: Borrowed reference. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key); - [DllImport(PythonDll)] + /// + /// Return 0 on success or -1 on failure. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItem(IntPtr pointer, IntPtr key, IntPtr value); - [DllImport(PythonDll)] - internal static extern int PyDict_SetItemString(IntPtr pointer, string key, IntPtr value); + /// + /// Return 0 on success or -1 on failure. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_SetItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key, IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_DelItem(IntPtr pointer, IntPtr key); - [DllImport(PythonDll)] - internal static extern int PyDict_DelItemString(IntPtr pointer, string key); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_DelItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyMapping_HasKey(IntPtr pointer, IntPtr key); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_Keys(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_Values(IntPtr pointer); - [DllImport(PythonDll)] - internal static extern IntPtr PyDict_Items(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyDict_Items(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_Copy(IntPtr pointer); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_Update(IntPtr pointer, IntPtr other); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyDict_Clear(IntPtr pointer); - [DllImport(PythonDll)] - internal static extern int PyDict_Size(IntPtr pointer); + internal static long PyDict_Size(IntPtr pointer) + { + return (long)_PyDict_Size(pointer); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] + internal static extern IntPtr _PyDict_Size(IntPtr pointer); + + + /// + /// Return value: New reference. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PySet_New(IntPtr iterable); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Add(IntPtr set, IntPtr key); + /// + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); //==================================================================== // Python list API @@ -1346,39 +1722,73 @@ internal static bool PyList_Check(IntPtr ob) return PyObject_TYPE(ob) == PyListType; } - [DllImport(PythonDll)] - internal static extern IntPtr PyList_New(int size); + internal static IntPtr PyList_New(long size) + { + return PyList_New(new IntPtr(size)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyList_New(IntPtr size); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - [DllImport(PythonDll)] - internal static extern IntPtr PyList_GetItem(IntPtr pointer, int index); + internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, long index) + { + return PyList_GetItem(pointer, new IntPtr(index)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern BorrowedReference PyList_GetItem(BorrowedReference pointer, IntPtr index); + + internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) + { + return PyList_SetItem(pointer, new IntPtr(index), value); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr value) + { + return PyList_Insert(pointer, new IntPtr(index), value); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value); - [DllImport(PythonDll)] - internal static extern int PyList_SetItem(IntPtr pointer, int index, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Append(BorrowedReference pointer, IntPtr value); - [DllImport(PythonDll)] - internal static extern int PyList_Insert(IntPtr pointer, int index, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Reverse(BorrowedReference pointer); - [DllImport(PythonDll)] - internal static extern int PyList_Append(IntPtr pointer, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Sort(BorrowedReference pointer); - [DllImport(PythonDll)] - internal static extern int PyList_Reverse(IntPtr pointer); + internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) + { + return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); + } - [DllImport(PythonDll)] - internal static extern int PyList_Sort(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - [DllImport(PythonDll)] - internal static extern IntPtr PyList_GetSlice(IntPtr pointer, int start, int end); + internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) + { + return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); + } - [DllImport(PythonDll)] - internal static extern int PyList_SetSlice(IntPtr pointer, int start, int end, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); - [DllImport(PythonDll)] - internal static extern int PyList_Size(IntPtr pointer); + internal static long PyList_Size(BorrowedReference pointer) + { + return (long)_PyList_Size(pointer); + } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] + private static extern IntPtr _PyList_Size(BorrowedReference pointer); //==================================================================== // Python tuple API @@ -1389,98 +1799,124 @@ internal static bool PyTuple_Check(IntPtr ob) return PyObject_TYPE(ob) == PyTupleType; } - [DllImport(PythonDll)] - internal static extern IntPtr PyTuple_New(int size); + internal static IntPtr PyTuple_New(long size) + { + return PyTuple_New(new IntPtr(size)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyTuple_New(IntPtr size); + + internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, long index) + => PyTuple_GetItem(pointer, new IntPtr(index)); + internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) + { + return PyTuple_GetItem(pointer, new IntPtr(index)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern BorrowedReference PyTuple_GetItem(BorrowedReference pointer, IntPtr index); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); + + internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) + { + return PyTuple_SetItem(pointer, new IntPtr(index), value); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - [DllImport(PythonDll)] - internal static extern IntPtr PyTuple_GetItem(IntPtr pointer, int index); + internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) + { + return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); + } - [DllImport(PythonDll)] - internal static extern int PyTuple_SetItem(IntPtr pointer, int index, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - [DllImport(PythonDll)] - internal static extern IntPtr PyTuple_GetSlice(IntPtr pointer, int start, int end); + internal static long PyTuple_Size(IntPtr pointer) + { + return (long)_PyTuple_Size(pointer); + } - [DllImport(PythonDll)] - internal static extern int PyTuple_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] + private static extern IntPtr _PyTuple_Size(IntPtr pointer); //==================================================================== // Python iterator API //==================================================================== -#if PYTHON2 - [DllImport(PythonDll)] - internal static extern bool PyIter_Check(IntPtr pointer); -#elif PYTHON3 internal static bool PyIter_Check(IntPtr pointer) { - var ob_type = (IntPtr)Marshal.PtrToStructure(pointer + ObjectOffset.ob_type, typeof(IntPtr)); - IntPtr tp_iternext = ob_type + TypeOffset.tp_iternext; - return tp_iternext != null && tp_iternext != _PyObject_NextNotImplemented; + var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + IntPtr tp_iternext = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iternext); + return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } -#endif - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyIter_Next(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyIter_Next(BorrowedReference pointer); //==================================================================== // Python module API //==================================================================== - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyModule_New(string name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern string PyModule_GetName(IntPtr module); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyModule_GetDict(IntPtr module); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern string PyModule_GetFilename(IntPtr module); -#if PYTHON3 - [DllImport(PythonDll)] - internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] +#else + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] #endif + internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_Import(IntPtr name); - [DllImport(PythonDll)] + /// + /// Return value: New reference. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ImportModule(string name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ReloadModule(IntPtr module); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_AddModule(string name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_GetModuleDict(); -#if PYTHON3 - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PySys_SetArgvEx( int argc, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv, int updatepath ); -#elif PYTHON2 - [DllImport(PythonDll)] - internal static extern void PySys_SetArgvEx( - int argc, - string[] argv, - int updatepath - ); -#endif - [DllImport(PythonDll)] - internal static extern IntPtr PySys_GetObject(string name); + /// + /// Return value: Borrowed reference. + /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern BorrowedReference PySys_GetObject(string name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySys_SetObject(string name, IntPtr ob); @@ -1493,63 +1929,89 @@ internal static bool PyType_Check(IntPtr ob) return PyObject_TypeCheck(ob, PyTypeType); } - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyType_Modified(IntPtr type); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PyType_IsSubtype(IntPtr t1, IntPtr t2); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2); internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) + => PyObject_TypeCheck(new BorrowedReference(ob), new BorrowedReference(tp)); + internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) { - IntPtr t = PyObject_TYPE(ob); + BorrowedReference t = PyObject_TYPE(ob); return (t == tp) || PyType_IsSubtype(t, tp); } - [DllImport(PythonDll)] + internal static bool PyType_IsSameAsOrSubtype(IntPtr type, IntPtr ofType) + { + return (type == ofType) || PyType_IsSubtype(type, ofType); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); - [DllImport(PythonDll)] - internal static extern IntPtr PyType_GenericAlloc(IntPtr type, int n); + internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) + { + return PyType_GenericAlloc(type, new IntPtr(n)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); - [DllImport(PythonDll)] + /// + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr _PyType_Lookup(IntPtr type, IntPtr name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr _PyObject_GetDictPtr(IntPtr obj); - [DllImport(PythonDll)] - internal static extern IntPtr PyObject_GC_New(IntPtr tp); - - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyObject_GC_Del(IntPtr tp); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyObject_GC_Track(IntPtr tp); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyObject_GC_UnTrack(IntPtr tp); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _PyObject_Dump(IntPtr ob); //==================================================================== // Python memory API //==================================================================== - [DllImport(PythonDll)] - internal static extern IntPtr PyMem_Malloc(int size); + internal static IntPtr PyMem_Malloc(long size) + { + return PyMem_Malloc(new IntPtr(size)); + } + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyMem_Malloc(IntPtr size); + + internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) + { + return PyMem_Realloc(ptr, new IntPtr(size)); + } - [DllImport(PythonDll)] - internal static extern IntPtr PyMem_Realloc(IntPtr ptr, int size); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyMem_Free(IntPtr ptr); @@ -1557,51 +2019,202 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) // Python exception API //==================================================================== - [DllImport(PythonDll)] - internal static extern void PyErr_SetString(IntPtr ob, string message); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyErr_SetString(IntPtr ob, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string message); - [DllImport(PythonDll)] - internal static extern void PyErr_SetObject(IntPtr ob, IntPtr message); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyErr_SetFromErrno(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_SetNone(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyErr_ExceptionMatches(IntPtr exception); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val); - [DllImport(PythonDll)] - internal static extern void PyErr_NormalizeException(IntPtr ob, IntPtr val, IntPtr tb); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); - [DllImport(PythonDll)] - internal static extern int PyErr_Occurred(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyErr_Occurred(); - [DllImport(PythonDll)] - internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Clear(); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + //==================================================================== + // Cell API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyCell_Get(BorrowedReference cell); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); + + //==================================================================== + // Python GC API + //==================================================================== + + internal const int _PyGC_REFS_SHIFT = 1; + internal const long _PyGC_REFS_UNTRACKED = -2; + internal const long _PyGC_REFS_REACHABLE = -3; + internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; + + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyGC_Collect(); + + internal static IntPtr _Py_AS_GC(IntPtr ob) + { + // XXX: PyGC_Head has a force alignment depend on platform. + // See PyGC_Head in objimpl.h for more details. + return Is32Bit ? ob - 16 : ob - 24; + } + + internal static IntPtr _Py_FROM_GC(IntPtr gc) + { + return Is32Bit ? gc + 16 : gc + 24; + } + + internal static IntPtr _PyGCHead_REFS(IntPtr gc) + { + unsafe + { + var pGC = (PyGC_Head*)gc; + var refs = pGC->gc.gc_refs; + if (Is32Bit) + { + return new IntPtr(refs.ToInt32() >> _PyGC_REFS_SHIFT); + } + return new IntPtr(refs.ToInt64() >> _PyGC_REFS_SHIFT); + } + } + + internal static IntPtr _PyGC_REFS(IntPtr ob) + { + return _PyGCHead_REFS(_Py_AS_GC(ob)); + } + + internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) + { + return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; + } + + internal static void Py_CLEAR(ref IntPtr ob) + { + XDecref(ob); + ob = IntPtr.Zero; + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer); //==================================================================== // Miscellaneous //==================================================================== - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Self(IntPtr ob); - [DllImport(PythonDll)] + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Function(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_MakePendingCalls(); + + internal static void SetNoSiteFlag() + { + var loader = LibraryLoader.Instance; + IntPtr dllLocal = IntPtr.Zero; + if (_PythonDll != "__Internal") + { + dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) + { + throw new Exception($"Cannot load {_PythonDll}"); + } + } + try + { + Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); + Marshal.WriteInt32(Py_NoSiteFlag, 1); + } + finally + { + if (dllLocal != IntPtr.Zero) + { + loader.Free(dllLocal); + } + } + } + + /// + /// Return value: New reference. + /// + internal static IntPtr GetBuiltins() + { + return PyImport_Import(PyIdentifier.builtins); + } + } + + + public enum ShutdownMode + { + Default, + Normal, + Soft, + Reload, + } + + + class PyReferenceCollection + { + private List> _actions = new List>(); + + /// + /// Record obj's address to release the obj in the future, + /// obj must alive before calling Release. + /// + public void Add(IntPtr ob, Action onRelease) + { + _actions.Add(new KeyValuePair(ob, onRelease)); + } + + public void Release() + { + foreach (var item in _actions) + { + Runtime.XDecref(item.Key); + item.Value?.Invoke(); + } + _actions.Clear(); + } } } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs new file mode 100644 index 000000000..f45e76db4 --- /dev/null +++ b/src/runtime/runtime_data.cs @@ -0,0 +1,447 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + public static class RuntimeData + { + private static Type _formatterType; + public static Type FormatterType + { + get => _formatterType; + set + { + if (!typeof(IFormatter).IsAssignableFrom(value)) + { + throw new ArgumentException("Not a type implemented IFormatter"); + } + _formatterType = value; + } + } + + public static ICLRObjectStorer WrappersStorer { get; set; } + + /// + /// Clears the old "clr_data" entry if a previous one is present. + /// + static void ClearCLRData () + { + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (!capsule.IsNull) + { + IntPtr oldData = PyCapsule_GetPointer(capsule, null); + PyMem_Free(oldData); + PyCapsule_SetPointer(capsule, IntPtr.Zero); + } + } + + internal static void Stash() + { + var metaStorage = new RuntimeDataStorage(); + MetaType.SaveRuntimeData(metaStorage); + + var importStorage = new RuntimeDataStorage(); + ImportHook.SaveRuntimeData(importStorage); + + var typeStorage = new RuntimeDataStorage(); + TypeManager.SaveRuntimeData(typeStorage); + + var clsStorage = new RuntimeDataStorage(); + ClassManager.SaveRuntimeData(clsStorage); + + var moduleStorage = new RuntimeDataStorage(); + SaveRuntimeDataModules(moduleStorage); + + var objStorage = new RuntimeDataStorage(); + SaveRuntimeDataObjects(objStorage); + + var runtimeStorage = new RuntimeDataStorage(); + runtimeStorage.AddValue("meta", metaStorage); + runtimeStorage.AddValue("import", importStorage); + runtimeStorage.AddValue("types", typeStorage); + runtimeStorage.AddValue("classes", clsStorage); + runtimeStorage.AddValue("modules", moduleStorage); + runtimeStorage.AddValue("objs", objStorage); + + IFormatter formatter = CreateFormatter(); + var ms = new MemoryStream(); + formatter.Serialize(ms, runtimeStorage); + + Debug.Assert(ms.Length <= int.MaxValue); + byte[] data = ms.GetBuffer(); + // TODO: use buffer api instead + IntPtr mem = PyMem_Malloc(ms.Length + IntPtr.Size); + Marshal.WriteIntPtr(mem, (IntPtr)ms.Length); + Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); + + ClearCLRData(); + NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); + PySys_SetObject("clr_data", capsule.DangerousGetAddress()); + // Let the dictionary own the reference + capsule.Dispose(); + } + + internal static void RestoreRuntimeData() + { + try + { + RestoreRuntimeDataImpl(); + } + finally + { + ClearStash(); + } + } + + private static void RestoreRuntimeDataImpl() + { + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (capsule.IsNull) + { + return; + } + IntPtr mem = PyCapsule_GetPointer(capsule, null); + int length = (int)Marshal.ReadIntPtr(mem); + byte[] data = new byte[length]; + Marshal.Copy(mem + IntPtr.Size, data, 0, length); + var ms = new MemoryStream(data); + var formatter = CreateFormatter(); + var storage = (RuntimeDataStorage)formatter.Deserialize(ms); + + var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); + RestoreRuntimeDataModules(storage.GetStorage("modules")); + TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); + ImportHook.RestoreRuntimeData(storage.GetStorage("import")); + PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); + + foreach (var item in objs) + { + item.Value.ExecutePostActions(); + XDecref(item.Key.pyHandle); + } + foreach (var item in clsObjs) + { + item.Value.ExecutePostActions(); + } + } + + public static bool HasStashData() + { + return !PySys_GetObject("clr_data").IsNull; + } + + public static void ClearStash() + { + PySys_SetObject("clr_data", IntPtr.Zero); + } + + static bool CheckSerializable (object o) + { + Type type = o.GetType(); + do + { + if (!type.IsSerializable) + { + return false; + } + } while ((type = type.BaseType) != null); + return true; + } + + private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) + { + var objs = ManagedType.GetManagedObjects(); + var extensionObjs = new List(); + var wrappers = new Dictionary>(); + var serializeObjs = new CLRWrapperCollection(); + var contexts = new Dictionary(); + foreach (var entry in objs) + { + var obj = entry.Key; + XIncref(obj.pyHandle); + switch (entry.Value) + { + case ManagedType.TrackTypes.Extension: + Debug.Assert(CheckSerializable(obj)); + var context = new InterDomainContext(); + contexts[obj.pyHandle] = context; + obj.Save(context); + extensionObjs.Add(obj); + break; + case ManagedType.TrackTypes.Wrapper: + // Wrapper must be the CLRObject + var clrObj = (CLRObject)obj; + object inst = clrObj.inst; + CLRMappedItem item; + List mappedObjs; + if (!serializeObjs.TryGetValue(inst, out item)) + { + item = new CLRMappedItem(inst) + { + Handles = new List() + }; + serializeObjs.Add(item); + + Debug.Assert(!wrappers.ContainsKey(inst)); + mappedObjs = new List(); + wrappers.Add(inst, mappedObjs); + } + else + { + mappedObjs = wrappers[inst]; + } + item.Handles.Add(clrObj.pyHandle); + mappedObjs.Add(clrObj); + break; + default: + break; + } + } + + var wrapperStorage = new RuntimeDataStorage(); + WrappersStorer?.Store(serializeObjs, wrapperStorage); + + var internalStores = new List(); + foreach (var item in serializeObjs) + { + if (!item.Stored) + { + if (!CheckSerializable(item.Instance)) + { + continue; + } + internalStores.AddRange(wrappers[item.Instance]); + } + foreach (var clrObj in wrappers[item.Instance]) + { + XIncref(clrObj.pyHandle); + var context = new InterDomainContext(); + contexts[clrObj.pyHandle] = context; + clrObj.Save(context); + } + } + storage.AddValue("internalStores", internalStores); + storage.AddValue("extensions", extensionObjs); + storage.AddValue("wrappers", wrapperStorage); + storage.AddValue("contexts", contexts); + } + + private static Dictionary RestoreRuntimeDataObjects(RuntimeDataStorage storage) + { + var extensions = storage.GetValue>("extensions"); + var internalStores = storage.GetValue>("internalStores"); + var contexts = storage.GetValue >("contexts"); + var storedObjs = new Dictionary(); + foreach (var obj in Enumerable.Union(extensions, internalStores)) + { + var context = contexts[obj.pyHandle]; + obj.Load(context); + storedObjs.Add(obj, context); + } + if (WrappersStorer != null) + { + var wrapperStorage = storage.GetStorage("wrappers"); + var handle2Obj = WrappersStorer.Restore(wrapperStorage); + foreach (var item in handle2Obj) + { + object obj = item.Instance; + foreach (var handle in item.Handles) + { + var context = contexts[handle]; + var co = CLRObject.Restore(obj, handle, context); + storedObjs.Add(co, context); + } + } + } + return storedObjs; + } + + private static void SaveRuntimeDataModules(RuntimeDataStorage storage) + { + var pyModules = PyImport_GetModuleDict(); + var items = PyDict_Items(pyModules); + long length = PyList_Size(items); + var modules = new Dictionary(); ; + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + if (ManagedType.IsManagedType(module)) + { + XIncref(name); + XIncref(module); + modules.Add(name, module); + } + } + items.Dispose(); + storage.AddValue("modules", modules); + } + + private static void RestoreRuntimeDataModules(RuntimeDataStorage storage) + { + var modules = storage.GetValue>("modules"); + var pyMoudles = PyImport_GetModuleDict(); + foreach (var item in modules) + { + int res = PyDict_SetItem(pyMoudles, item.Key, item.Value); + PythonException.ThrowIfIsNotZero(res); + XDecref(item.Key); + XDecref(item.Value); + } + modules.Clear(); + } + + private static IFormatter CreateFormatter() + { + return FormatterType != null ? + (IFormatter)Activator.CreateInstance(FormatterType) + : new BinaryFormatter(); + } + } + + + [Serializable] + public class RuntimeDataStorage + { + private Stack _stack; + private Dictionary _namedValues; + + public T AddValue(string name, T value) + { + if (_namedValues == null) + { + _namedValues = new Dictionary(); + } + _namedValues.Add(name, value); + return value; + } + + public object GetValue(string name) + { + return _namedValues[name]; + } + + public T GetValue(string name) + { + return (T)GetValue(name); + } + + public T GetValue(string name, out T value) + { + value = GetValue(name); + return value; + } + + public RuntimeDataStorage GetStorage(string name) + { + return GetValue(name); + } + + public T PushValue(T value) + { + if (_stack == null) + { + _stack = new Stack(); + } + _stack.Push(value); + return value; + } + + public object PopValue() + { + return _stack.Pop(); + } + + public T PopValue() + { + return (T)PopValue(); + } + + public T PopValue(out T value) + { + return value = (T)PopValue(); + } + } + + + [Serializable] + class InterDomainContext + { + private RuntimeDataStorage _storage; + public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); + + /// + /// Actions after loaded. + /// + [NonSerialized] + private List _postActions; + public List PostActions => _postActions ?? (_postActions = new List()); + + public void AddPostAction(Action action) + { + PostActions.Add(action); + } + + public void ExecutePostActions() + { + if (_postActions == null) + { + return; + } + foreach (var action in _postActions) + { + action(); + } + } + } + + public class CLRMappedItem + { + public object Instance { get; private set; } + public IList Handles { get; set; } + public bool Stored { get; set; } + + public CLRMappedItem(object instance) + { + Instance = instance; + } + } + + + public interface ICLRObjectStorer + { + ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); + CLRWrapperCollection Restore(RuntimeDataStorage storage); + } + + + public class CLRWrapperCollection : KeyedCollection + { + public bool TryGetValue(object key, out CLRMappedItem value) + { + if (Dictionary == null) + { + value = null; + return false; + } + return Dictionary.TryGetValue(key, out value); + } + + protected override object GetKeyForItem(CLRMappedItem item) + { + return item.Instance; + } + } +} diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs new file mode 100644 index 000000000..69acbcd31 --- /dev/null +++ b/src/runtime/runtime_state.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + class RuntimeState + { + public static bool ShouldRestoreObjects { get; set; } = false; + public static bool UseDummyGC { get; set; } = false; + + public static void Save() + { + if (!PySys_GetObject("dummy_gc").IsNull) + { + throw new Exception("Runtime State set already"); + } + + IntPtr objs = IntPtr.Zero; + if (ShouldRestoreObjects) + { + objs = PySet_New(IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + AddObjPtrToSet(objs, obj); + } + } + + var modules = PySet_New(IntPtr.Zero); + foreach (var name in GetModuleNames()) + { + int res = PySet_Add(modules, name); + PythonException.ThrowIfIsNotZero(res); + } + + + var dummyGCHead = PyMem_Malloc(Marshal.SizeOf(typeof(PyGC_Head))); + unsafe + { + var head = (PyGC_Head*)dummyGCHead; + head->gc.gc_next = dummyGCHead; + head->gc.gc_prev = dummyGCHead; + head->gc.gc_refs = IntPtr.Zero; + } + { + var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); + int res = PySys_SetObject("dummy_gc", pyDummyGC); + PythonException.ThrowIfIsNotZero(res); + XDecref(pyDummyGC); + + try + { + res = PySys_SetObject("initial_modules", modules); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(modules); + } + + if (ShouldRestoreObjects) + { + AddObjPtrToSet(objs, modules); + try + { + res = PySys_SetObject("initial_objs", objs); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(objs); + } + } + } + } + + public static void Restore() + { + var dummyGCAddr = PySys_GetObject("dummy_gc").DangerousGetAddress(); + if (dummyGCAddr == IntPtr.Zero) + { + throw new InvalidOperationException("Runtime state have not set"); + } + var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); + ResotreModules(dummyGC); + if (ShouldRestoreObjects) + { + RestoreObjects(dummyGC); + } + } + + private static void ResotreModules(IntPtr dummyGC) + { + var intialModules = PySys_GetObject("initial_modules"); + Debug.Assert(!intialModules.IsNull); + var modules = PyImport_GetModuleDict(); + foreach (var name in GetModuleNames()) + { + if (PySet_Contains(intialModules.DangerousGetAddress(), name) == 1) + { + continue; + } + var module = PyDict_GetItem(modules, name); + + if (UseDummyGC && _PyObject_GC_IS_TRACKED(module)) + { + ExchangeGCChain(module, dummyGC); + } + if (PyDict_DelItem(modules, name) != 0) + { + PyErr_Print(); + } + } + } + + private static void RestoreObjects(IntPtr dummyGC) + { + if (!UseDummyGC) + { + throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); + } + IntPtr intialObjs = PySys_GetObject("initial_objs").DangerousGetAddress(); + Debug.Assert(intialObjs != IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + var p = PyLong_FromVoidPtr(obj); + try + { + if (PySet_Contains(intialObjs, p) == 1) + { + continue; + } + } + finally + { + XDecref(p); + } + Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); + ExchangeGCChain(obj, dummyGC); + } + } + + public static IEnumerable PyGCGetObjects() + { + var gc = PyImport_ImportModule("gc"); + PythonException.ThrowIfIsNull(gc); + var get_objects = PyObject_GetAttrString(gc, "get_objects"); + var objs = PyObject_CallObject(get_objects, IntPtr.Zero); + var length = PyList_Size(new BorrowedReference(objs)); + for (long i = 0; i < length; i++) + { + var obj = PyList_GetItem(new BorrowedReference(objs), i); + yield return obj.DangerousGetAddress(); + } + XDecref(objs); + XDecref(gc); + } + + public static IEnumerable GetModuleNames() + { + var modules = PyImport_GetModuleDict(); + var names = PyDict_Keys(modules); + var length = PyList_Size(new BorrowedReference(names)); + for (int i = 0; i < length; i++) + { + var name = PyList_GetItem(new BorrowedReference(names), i); + yield return name.DangerousGetAddress(); + } + XDecref(names); + } + + private static void AddObjPtrToSet(IntPtr set, IntPtr obj) + { + var p = PyLong_FromVoidPtr(obj); + XIncref(obj); + try + { + int res = PySet_Add(set, p); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(p); + } + } + /// + /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. + /// + private static void ExchangeGCChain(IntPtr obj, IntPtr gc) + { + var head = _Py_AS_GC(obj); + if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) + { + throw new ArgumentException("GC object untracked"); + } + unsafe + { + var g = (PyGC_Head*)head; + var newGCGen = (PyGC_Head*)gc; + + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = g->gc.gc_next; + ((PyGC_Head*)g->gc.gc_next)->gc.gc_prev = g->gc.gc_prev; + + g->gc.gc_next = gc; + g->gc.gc_prev = newGCGen->gc.gc_prev; + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = head; + newGCGen->gc.gc_prev = head; + } + } + + private static IEnumerable IterGCNodes(IntPtr gc) + { + var node = GetNextGCNode(gc); + while (node != gc) + { + var next = GetNextGCNode(node); + yield return node; + node = next; + } + } + + private static IEnumerable IterObjects(IntPtr gc) + { + foreach (var node in IterGCNodes(gc)) + { + yield return _Py_FROM_GC(node); + } + } + + private static unsafe IntPtr GetNextGCNode(IntPtr node) + { + return ((PyGC_Head*)node)->gc.gc_next; + } + } +} diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs new file mode 100644 index 000000000..a13c7b6f8 --- /dev/null +++ b/src/runtime/slots/mp_length.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; + +namespace Python.Runtime.Slots +{ + internal static class mp_length_slot + { + private static MethodInfo _lengthMethod; + public static MethodInfo Method + { + get + { + if (_lengthMethod != null) + { + return _lengthMethod; + } + _lengthMethod = typeof(mp_length_slot).GetMethod( + nameof(mp_length_slot.mp_length), + BindingFlags.Static | BindingFlags.NonPublic); + Debug.Assert(_lengthMethod != null); + return _lengthMethod; + } + } + + public static bool CanAssign(Type clrType) + { + if (typeof(ICollection).IsAssignableFrom(clrType)) + { + return true; + } + if (clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return true; + } + if (clrType.IsInterface && clrType.IsGenericType && clrType.GetGenericTypeDefinition() == typeof(ICollection<>)) + { + return true; + } + return false; + } + + /// + /// Implements __len__ for classes that implement ICollection + /// (this includes any IList implementer or Array subclass) + /// + private static int mp_length(IntPtr ob) + { + var co = ManagedType.GetManagedObject(ob) as CLRObject; + if (co == null) + { + Exceptions.RaiseTypeError("invalid object"); + } + + // first look for ICollection implementation directly + if (co.inst is ICollection c) + { + return c.Count; + } + + Type clrType = co.inst.GetType(); + + // now look for things that implement ICollection directly (non-explicitly) + PropertyInfo p = clrType.GetProperty("Count"); + if (p != null && clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return (int)p.GetValue(co.inst, null); + } + + // finally look for things that implement the interface explicitly + var iface = clrType.GetInterfaces().FirstOrDefault(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>)); + if (iface != null) + { + p = iface.GetProperty(nameof(ICollection.Count)); + return (int)p.GetValue(co.inst, null); + } + + Exceptions.SetError(Exceptions.TypeError, $"object of type '{clrType.Name}' has no len()"); + return -1; + } + } +} diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 6f373f036..973a5aea2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,28 +1,102 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using System.Diagnostics; +using Python.Runtime.Slots; +using static Python.Runtime.PythonException; namespace Python.Runtime { + /// /// The TypeManager class is responsible for building binary-compatible /// Python type objects that are implemented in managed code. /// internal class TypeManager { - private static BindingFlags tbFlags; - private static Dictionary cache; + internal static IntPtr subtype_traverse; + internal static IntPtr subtype_clear; + + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static Dictionary cache = new Dictionary(); + + private static readonly Dictionary _slotsHolders = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); + + // Slots which must be set + private static readonly string[] _requiredSlots = new string[] + { + "tp_traverse", + "tp_clear", + }; - static TypeManager() + internal static void Initialize() { - tbFlags = BindingFlags.Public | BindingFlags.Static; - cache = new Dictionary(128); + Debug.Assert(cache.Count == 0, "Cache should be empty", + "Some errors may occurred on last shutdown"); + IntPtr type = SlotHelper.CreateObjectType(); + subtype_traverse = Marshal.ReadIntPtr(type, TypeOffset.tp_traverse); + subtype_clear = Marshal.ReadIntPtr(type, TypeOffset.tp_clear); + Runtime.XDecref(type); } + internal static void RemoveTypes() + { + foreach (var tpHandle in cache.Values) + { + SlotsHolder holder; + if (_slotsHolders.TryGetValue(tpHandle, out holder)) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + holder.ResetSlots(); + } + } + Runtime.XDecref(tpHandle); + } + cache.Clear(); + _slotsImpls.Clear(); + _slotsHolders.Clear(); + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + foreach (var tpHandle in cache.Values) + { + Runtime.XIncref(tpHandle); + } + storage.AddValue("cache", cache); + storage.AddValue("slots", _slotsImpls); + } + + internal static void RestoreRuntimeData(RuntimeDataStorage storage) + { + Debug.Assert(cache == null || cache.Count == 0); + storage.GetValue("slots", out _slotsImpls); + storage.GetValue>("cache", out var _cache); + foreach (var entry in _cache) + { + if (!entry.Key.Valid) + { + Runtime.XDecref(entry.Value); + continue; + } + Type type = entry.Key.Value;; + IntPtr handle = entry.Value; + cache[type] = handle; + SlotsHolder holder = CreateSolotsHolder(handle); + InitializeSlots(handle, _slotsImpls[type], holder); + // FIXME: mp_length_slot.CanAssgin(clrType) + } + } /// + /// Return value: Borrowed reference. /// Given a managed Type derived from ExtensionType, get the handle to /// a Python type object that delegates its implementation to the Type /// object. These Python type instances are used to implement internal @@ -40,11 +114,13 @@ internal static IntPtr GetTypeHandle(Type type) } handle = CreateType(type); cache[type] = handle; + _slotsImpls.Add(type, type); return handle; } /// + /// Return value: Borrowed reference. /// Get the handle of a Python type that reflects the given CLR type. /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. @@ -59,6 +135,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) } handle = CreateType(obj, type); cache[type] = handle; + _slotsImpls.Add(type, obj.GetType()); return handle; } @@ -73,29 +150,37 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// internal static IntPtr CreateType(Type impl) { - IntPtr type = AllocateTypeObject(impl.Name); + IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); int ob_size = ObjectOffset.Size(type); // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.DictOffset(type); + var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + Util.WriteCLong(type, TypeOffset.tp_flags, flags); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); + Runtime.XDecref(mod); InitMethods(type, impl); + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } @@ -103,7 +188,7 @@ internal static IntPtr CreateType(Type impl) internal static IntPtr CreateType(ManagedType impl, Type clrType) { // Cleanup the type name to get rid of funny nested type names. - string name = "CLR." + clrType.FullName; + string name = $"clr.{clrType.FullName}"; int i = name.LastIndexOf('+'); if (i > -1) { @@ -117,7 +202,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) IntPtr base_ = IntPtr.Zero; int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - int tp_dictoffset = ObjectOffset.DictOffset(Runtime.PyTypeType); // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* @@ -125,9 +209,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (typeof(Exception).IsAssignableFrom(clrType)) { ob_size = ObjectOffset.Size(Exceptions.Exception); - tp_dictoffset = ObjectOffset.DictOffset(Exceptions.Exception); } + int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; + if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -138,7 +223,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) base_ = bc.pyHandle; } - IntPtr type = AllocateTypeObject(name); + IntPtr type = AllocateTypeObject(name, Runtime.PyCLRMetaType); Marshal.WriteIntPtr(type, TypeOffset.ob_type, Runtime.PyCLRMetaType); Runtime.XIncref(Runtime.PyCLRMetaType); @@ -147,7 +232,44 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); - InitializeSlots(type, impl.GetType()); + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + + if (Marshal.ReadIntPtr(type, TypeOffset.mp_length) == IntPtr.Zero + && mp_length_slot.CanAssign(clrType)) + { + InitializeSlot(type, TypeOffset.mp_length, mp_length_slot.Method, slotsHolder); + } + + if (!typeof(IEnumerable).IsAssignableFrom(clrType) && + !typeof(IEnumerator).IsAssignableFrom(clrType)) + { + // The tp_iter slot should only be set for enumerable types. + Marshal.WriteIntPtr(type, TypeOffset.tp_iter, IntPtr.Zero); + } + + + // Only set mp_subscript and mp_ass_subscript for types with indexers + if (impl is ClassBase cb) + { + if (!(impl is ArrayObject)) + { + if (cb.indexer == null || !cb.indexer.CanGet) + { + Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); + } + if (cb.indexer == null || !cb.indexer.CanSet) + { + Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); + } + } + } + else + { + Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); + Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); + } if (base_ != IntPtr.Zero) { @@ -155,31 +277,35 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.BaseType; - flags |= TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.BaseType + | TypeFlags.HaveGC; + Util.WriteCLong(type, TypeOffset.tp_flags, flags); + OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; IntPtr mod = Runtime.PyString_FromString(mn); - Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); + Runtime.XDecref(mod); // Hide the gchandle of the implementation in a magic type slot. - GCHandle gc = GCHandle.Alloc(impl); + GCHandle gc = impl.AllocGCHandle(); Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); // Set the handle attributes on the implementing instance. - impl.tpHandle = Runtime.PyCLRMetaType; - impl.gcHandle = gc; + impl.tpHandle = type; impl.pyHandle = type; //DebugUtil.DumpType(type); @@ -189,6 +315,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { + var dictRef = new BorrowedReference(py_dict); // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation string name = Runtime.GetManagedString(py_name); @@ -198,38 +325,29 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr object assembly = null; object namespaceStr = null; - var disposeList = new List(); - try + using (var assemblyKey = new PyString("__assembly__")) { - var assemblyKey = new PyObject(Converter.ToPython("__assembly__", typeof(string))); - disposeList.Add(assemblyKey); - if (0 != Runtime.PyMapping_HasKey(py_dict, assemblyKey.Handle)) + var assemblyPtr = Runtime.PyDict_GetItemWithError(dictRef, assemblyKey.Reference); + if (assemblyPtr.IsNull) { - var pyAssembly = new PyObject(Runtime.PyDict_GetItem(py_dict, assemblyKey.Handle)); - disposeList.Add(pyAssembly); - if (!Converter.ToManagedValue(pyAssembly.Handle, typeof(string), out assembly, false)) - { - throw new InvalidCastException("Couldn't convert __assembly__ value to string"); - } + if (Exceptions.ErrorOccurred()) return IntPtr.Zero; + } + else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, false)) + { + return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); } - var namespaceKey = new PyObject(Converter.ToPythonImplicit("__namespace__")); - disposeList.Add(namespaceKey); - if (0 != Runtime.PyMapping_HasKey(py_dict, namespaceKey.Handle)) + using (var namespaceKey = new PyString("__namespace__")) { - var pyNamespace = new PyObject(Runtime.PyDict_GetItem(py_dict, namespaceKey.Handle)); - disposeList.Add(pyNamespace); - if (!Converter.ToManagedValue(pyNamespace.Handle, typeof(string), out namespaceStr, false)) + var pyNamespace = Runtime.PyDict_GetItemWithError(dictRef, namespaceKey.Reference); + if (pyNamespace.IsNull) { - throw new InvalidCastException("Couldn't convert __namespace__ value to string"); + if (Exceptions.ErrorOccurred()) return IntPtr.Zero; + } + else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, false)) + { + return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); } - } - } - finally - { - foreach (PyObject o in disposeList) - { - o.Dispose(); } } @@ -243,7 +361,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr try { Type subType = ClassDerivedObject.CreateDerivedType(name, - baseClass.type, + baseClass.type.Value, py_dict, (string)namespaceStr, (string)assembly); @@ -255,7 +373,15 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr // by default the class dict will have all the C# methods in it, but as this is a // derived class we want the python overrides in there instead if they exist. IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); - Runtime.PyDict_Update(cls_dict, py_dict); + ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, py_dict)); + Runtime.XIncref(py_type); + // Update the __classcell__ if it exists + var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); + if (!cell.IsNull) + { + ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); + ThrowIfIsNotZero(Runtime.PyDict_DelItemString(cls_dict, "__classcell__")); + } return py_type; } @@ -288,80 +414,133 @@ internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); } - internal static IntPtr CreateMetaType(Type impl) + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the // standard Python metatype (PyType_Type). It overrides certain of // the standard type slots, and has to subclass PyType_Type for // certain functions in the C runtime to work correctly with it. - IntPtr type = AllocateTypeObject("CLR Metatype"); - IntPtr py_type = Runtime.PyTypeType; + IntPtr type = AllocateTypeObject("CLR Metatype", metatype: Runtime.PyTypeType); + IntPtr py_type = Runtime.PyTypeType; Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - // Copy gc and other type slots from the base Python metatype. + int size = TypeOffset.magic() + IntPtr.Size; + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); - CopySlot(py_type, type, TypeOffset.tp_basicsize); - CopySlot(py_type, type, TypeOffset.tp_itemsize); + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.HaveGC; + Util.WriteCLong(type, TypeOffset.tp_flags, flags); - CopySlot(py_type, type, TypeOffset.tp_dictoffset); - CopySlot(py_type, type, TypeOffset.tp_weaklistoffset); + // Slots will inherit from TypeType, it's not neccesary for setting them. + // Inheried slots: + // tp_basicsize, tp_itemsize, + // tp_dictoffset, tp_weaklistoffset, + // tp_traverse, tp_clear, tp_is_gc, etc. + slotsHolder = SetupMetaSlots(impl, type); - CopySlot(py_type, type, TypeOffset.tp_traverse); - CopySlot(py_type, type, TypeOffset.tp_clear); - CopySlot(py_type, type, TypeOffset.tp_is_gc); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } - // Override type slots with those of the managed implementation. + IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + IntPtr mod = Runtime.PyString_FromString("CLR"); + Runtime.PyDict_SetItemString(dict, "__module__", mod); - InitializeSlots(type, impl); + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); + //DebugUtil.DumpType(type); - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + return type; + } + + internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) + { + // Override type slots with those of the managed implementation. + SlotsHolder slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); - // We need space for 3 PyMethodDef structs, each of them - // 4 int-ptrs in size. - IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); + // We need space for 3 PyMethodDef structs. + int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); + IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); IntPtr mdefStart = mdef; - mdef = WriteMethodDef( - mdef, - "__instancecheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc") - ); - - mdef = WriteMethodDef( - mdef, - "__subclasscheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc") - ); - - // FIXME: mdef is not used + foreach (var methodName in MetaType.CustomMethods) + { + mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); + } mdef = WriteMethodDefSentinel(mdef); + Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - Runtime.PyType_Ready(type); - - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(dict, "__module__", mod); + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) + { + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); + }); + } + return slotsHolder; + } - //DebugUtil.DumpType(type); + private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, SlotsHolder slotsHolder) + { + MethodInfo mi = typeof(MetaType).GetMethod(name); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + slotsHolder.KeeapAlive(thunkInfo); - return type; + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + Debug.Fail($"Cannot remove {name} from metatype"); + } + FreeMethodDef(mdefAddr); + }); + } + mdef = WriteMethodDef(mdef, name, thunkInfo.Address); + return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) { // Utility to create a subtype of a std Python type, but with // a managed type able to override implementation - IntPtr type = AllocateTypeObject(name); + IntPtr type = AllocateTypeObject(name, metatype: Runtime.PyTypeType); //Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); //Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); @@ -378,19 +557,27 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; flags |= TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + Util.WriteCLong(type, TypeOffset.tp_flags, flags); CopySlot(base_, type, TypeOffset.tp_traverse); CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl, slotsHolder); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(tp_dict, "__module__", mod); + Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } @@ -399,61 +586,49 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) /// /// Utility method to allocate a type object & do basic initialization. /// - internal static IntPtr AllocateTypeObject(string name) + internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { - IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyTypeType, 0); + IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. -#if PYTHON3 - // For python3 we leak two objects. One for the ASCII representation - // required for tp_name, and another for the Unicode representation - // for ht_name. - IntPtr temp = Runtime.PyBytes_FromString(name); - IntPtr raw = Runtime.PyBytes_AS_STRING(temp); - temp = Runtime.PyUnicode_FromString(name); -#elif PYTHON2 - IntPtr temp = Runtime.PyString_FromString(name); - IntPtr raw = Runtime.PyString_AsString(temp); -#endif + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); -#if PYTHON3 + Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); -#endif - - long ptr = type.ToInt64(); // 64-bit safe - - temp = new IntPtr(ptr + TypeOffset.nb_add); + temp = type + TypeOffset.nb_add; Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); - temp = new IntPtr(ptr + TypeOffset.sq_length); + temp = type + TypeOffset.sq_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); - temp = new IntPtr(ptr + TypeOffset.mp_length); + temp = type + TypeOffset.mp_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); -#if PYTHON3 - temp = new IntPtr(ptr + TypeOffset.bf_getbuffer); -#elif PYTHON2 - temp = new IntPtr(ptr + TypeOffset.bf_getreadbuffer); -#endif + temp = type + TypeOffset.bf_getbuffer; Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); return type; } - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of /// the Python object to the managed methods of the implementing Type. /// - internal static void InitializeSlots(IntPtr type, Type impl) + internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHolder = null) { - var seen = new Hashtable(8); - Type offsetType = typeof(TypeOffset); + // We work from the most-derived class up; make sure to get + // the most-derived slot and not to override it with a base + // class's slot. + var seen = new HashSet(); while (impl != null) { @@ -461,34 +636,87 @@ internal static void InitializeSlots(IntPtr type, Type impl) foreach (MethodInfo method in methods) { string name = method.Name; - if (!(name.StartsWith("tp_") || - name.StartsWith("nb_") || - name.StartsWith("sq_") || - name.StartsWith("mp_") || - name.StartsWith("bf_") - )) + if (!name.StartsWith("tp_") && !TypeOffset.IsSupportedSlotName(name)) { + Debug.Assert(!name.Contains("_") || name.StartsWith("_") || method.IsSpecialName); continue; } - if (seen[name] != null) + if (seen.Contains(name)) { continue; } - FieldInfo fi = offsetType.GetField(name); - var offset = (int)fi.GetValue(offsetType); + InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); - IntPtr slot = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, offset, slot); - - seen[name] = 1; + seen.Add(name); } impl = impl.BaseType; } + + foreach (string slot in _requiredSlots) + { + if (seen.Contains(slot)) + { + continue; + } + var offset = ManagedDataOffsets.GetSlotOffset(slot); + Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); + } } + /// + /// Helper for InitializeSlots. + /// + /// Initializes one slot to point to a function pointer. + /// The function pointer might be a thunk for C#, or it may be + /// an address in the NativeCodePage. + /// + /// Type being initialized. + /// Function pointer. + /// Name of the method. + /// Can override the slot when it existed + static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) + { + var offset = ManagedDataOffsets.GetSlotOffset(name); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, slot); + } + + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) + { + int offset = ManagedDataOffsets.GetSlotOffset(name); + + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(offset, thunk); + } + } + + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder = null) + { + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(slotOffset, thunk); + } + } + + static bool IsSlotSet(IntPtr type, string name) + { + int offset = ManagedDataOffsets.GetSlotOffset(name); + return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; + } /// /// Given a newly allocated Python type object and a managed Type that @@ -518,6 +746,7 @@ private static void InitMethods(IntPtr pytype, Type type) mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + m.DecrRefCount(); addedMethods.Add(method_name); } } @@ -535,5 +764,181 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) IntPtr fp = Marshal.ReadIntPtr(from, offset); Marshal.WriteIntPtr(to, offset, fp); } + + private static SlotsHolder CreateSolotsHolder(IntPtr type) + { + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + + class SlotsHolder + { + public delegate void Resetor(IntPtr type, int offset); + + private readonly IntPtr _type; + private Dictionary _slots = new Dictionary(); + private List _keepalive = new List(); + private Dictionary _customResetors = new Dictionary(); + private List _deallocators = new List(); + private bool _alreadyReset = false; + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(IntPtr type) + { + _type = type; + } + + public void Set(int offset, ThunkInfo thunk) + { + _slots[offset] = thunk; + } + + public void Set(int offset, Resetor resetor) + { + _customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + _deallocators.Add(deallocate); + } + + public void KeeapAlive(ThunkInfo thunk) + { + _keepalive.Add(thunk); + } + + public void ResetSlots() + { + if (_alreadyReset) + { + return; + } + _alreadyReset = true; +#if DEBUG + IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + foreach (var offset in _slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Marshal.WriteIntPtr(_type, offset, ptr); + } + + foreach (var action in _deallocators) + { + action(); + } + + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(_type, offset); + } + + _customResetors.Clear(); + _slots.Clear(); + _keepalive.Clear(); + _deallocators.Clear(); + + // Custom reset + IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); + if (handlePtr != IntPtr.Zero) + { + GCHandle handle = GCHandle.FromIntPtr(handlePtr); + if (handle.IsAllocated) + { + handle.Free(); + } + Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); + } + } + + public static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear) + { + return TypeManager.subtype_clear; + } + else if (offset == TypeOffset.tp_traverse) + { + return TypeManager.subtype_traverse; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObejct_GC_Del. + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + IntPtr res = resRef.DangerousGetAddress(); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + resRef.Dispose(); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } } } diff --git a/src/testing/InheritanceTest.cs b/src/testing/InheritanceTest.cs new file mode 100644 index 000000000..529703b3c --- /dev/null +++ b/src/testing/InheritanceTest.cs @@ -0,0 +1,14 @@ +using System; + +namespace Python.Test +{ + public class BaseClass + { + public bool IsBase() => true; + } + + public class DerivedClass : BaseClass + { + public new bool IsBase() => false; + } +} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index ce8dca10d..e0c0ca8b1 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,113 +1,13 @@ - - + - Debug - AnyCPU - {6F401A34-273B-450F-9A4C-13550BE0767B} - Library - Python.Test - Python.Test - bin\Python.Test.xml - bin\ - v4.0 - - 1591,0067 - ..\..\ - $(SolutionDir)\bin\ - 6 - false - ..\pythonnet.snk - prompt + netstandard2.0 - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - - - - - - - - - - - - - - - - - - - - - + - - + - - - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Python.Runtime - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - + - - + diff --git a/src/testing/ReprTest.cs b/src/testing/ReprTest.cs new file mode 100644 index 000000000..48e93683a --- /dev/null +++ b/src/testing/ReprTest.cs @@ -0,0 +1,108 @@ +using System; +using System.Text; + +namespace Python.Test +{ + /// + /// Supports repr unit tests. + /// + public class ReprTest + { + public class Point + { + public Point(double x, double y) + { + X = x; + Y = y; + } + + public double X { get; set; } + public double Y { get; set; } + + public override string ToString() + { + return base.ToString() + ": X=" + X.ToString() + ", Y=" + Y.ToString(); + } + + public string __repr__() + { + return "Point(" + X.ToString() + "," + Y.ToString() + ")"; + } + } + + public class Foo + { + public string __repr__() + { + return "I implement __repr__() but not ToString()!"; + } + } + + public class Bar + { + public override string ToString() + { + return "I implement ToString() but not __repr__()!"; + } + } + + public class BazBase + { + public override string ToString() + { + return "Base class implementing ToString()!"; + } + } + + public class BazMiddle : BazBase + { + public override string ToString() + { + return "Middle class implementing ToString()!"; + } + } + + //implements ToString via BazMiddle + public class Baz : BazMiddle + { + + } + + public class Quux + { + public string ToString(string format) + { + return "I implement ToString() with an argument!"; + } + } + + public class QuuzBase + { + protected string __repr__() + { + return "I implement __repr__ but it isn't public!"; + } + } + + public class Quuz : QuuzBase + { + + } + + public class Corge + { + public string __repr__(int i) + { + return "__repr__ implemention with input parameter!"; + } + } + + public class Grault + { + public int __repr__() + { + return "__repr__ implemention with wrong return type!".Length; + } + } + } +} diff --git a/src/testing/constructortests.cs b/src/testing/constructortests.cs index 8800c94a1..4dc7f04d8 100644 --- a/src/testing/constructortests.cs +++ b/src/testing/constructortests.cs @@ -48,4 +48,22 @@ public SubclassConstructorTest(Exception v) value = v; } } + + public class MultipleConstructorsTest + { + public string value; + public Type[] type; + + public MultipleConstructorsTest() + { + value = ""; + type = new Type[1] { null }; + } + + public MultipleConstructorsTest(string s, params Type[] tp) + { + value = s; + type = tp; + } + } } diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 7179c7607..1f9d64e1b 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -1,7 +1,9 @@ namespace Python.Test { + using System.Collections.Generic; + /// - /// Supports units tests for field access. + /// Supports unit tests for field access. /// public class ConversionTest { @@ -32,8 +34,15 @@ public ConversionTest() public byte[] ByteArrayField; public sbyte[] SByteArrayField; + public readonly List ListField = new List(); + + public T? Echo(T? arg) where T: struct { + return arg; + } + } + public interface ISpam { @@ -54,4 +63,19 @@ public string GetValue() return value; } } + + public class UnicodeString + { + public string value = "안녕"; + + public string GetString() + { + return value; + } + + public override string ToString() + { + return value; + } + } } diff --git a/src/testing/eventtest.cs b/src/testing/eventtest.cs index 4c701d488..dfbd5c881 100644 --- a/src/testing/eventtest.cs +++ b/src/testing/eventtest.cs @@ -8,6 +8,7 @@ namespace Python.Test public delegate void EventHandlerTest(object sender, EventArgsTest e); + #pragma warning disable 67 // Unused events, these are only accessed from Python public class EventTest { public static event EventHandlerTest PublicStaticEvent; @@ -100,6 +101,7 @@ public static void ShutUpCompiler() e.PrivateEvent += f; } } + #pragma warning restore 67 public class EventArgsTest : EventArgs diff --git a/src/testing/exceptiontest.cs b/src/testing/exceptiontest.cs index e4f683721..45acaa2cc 100644 --- a/src/testing/exceptiontest.cs +++ b/src/testing/exceptiontest.cs @@ -1,4 +1,6 @@ using System; +using System.Collections; +using System.Collections.Generic; namespace Python.Test { @@ -54,6 +56,13 @@ public static bool ThrowException() throw new OverflowException("error"); } + public static IEnumerable ThrowExceptionInIterator(Exception e) + { + yield return 1; + yield return 2; + throw e; + } + public static void ThrowChainedExceptions() { try diff --git a/src/testing/indexertest.cs b/src/testing/indexertest.cs index ae39eb2b1..78bb99a7c 100644 --- a/src/testing/indexertest.cs +++ b/src/testing/indexertest.cs @@ -411,4 +411,29 @@ public MultiDefaultKeyIndexerTest() : base() } } } + + public class PublicInheritedIndexerTest : PublicIndexerTest { } + + public class ProtectedInheritedIndexerTest : ProtectedIndexerTest { } + + public class PrivateInheritedIndexerTest : ProtectedIndexerTest { } + + public class InternalInheritedIndexerTest : InternalIndexerTest { } + + public interface IIndexer + { + string this[int index] { get; set; } + } + + public interface IInheritedIndexer : IIndexer { } + + public class InterfaceInheritedIndexerTest :IndexerBase, IInheritedIndexer { + private System.Collections.Generic.IDictionary d = new System.Collections.Generic.Dictionary(); + + public string this[int index] + { + get { return GetValue(index); } + set { t[index] = value; } + } + } } diff --git a/src/testing/interfacetest.cs b/src/testing/interfacetest.cs index 2c24596bc..0158d64da 100644 --- a/src/testing/interfacetest.cs +++ b/src/testing/interfacetest.cs @@ -11,7 +11,6 @@ internal interface IInternalInterface { } - public interface ISayHello1 { string SayHello(); @@ -43,6 +42,27 @@ string ISayHello2.SayHello() return "hello 2"; } + public ISayHello1 GetISayHello1() + { + return this; + } + + public void GetISayHello2(out ISayHello2 hello2) + { + hello2 = this; + } + + public ISayHello1 GetNoSayHello(out ISayHello2 hello2) + { + hello2 = null; + return null; + } + + public ISayHello1 [] GetISayHello1Array() + { + return new[] { this }; + } + public interface IPublic { } diff --git a/src/testing/methodtest.cs b/src/testing/methodtest.cs index 83dc907c0..f5d694488 100644 --- a/src/testing/methodtest.cs +++ b/src/testing/methodtest.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Linq; +using System.Runtime.InteropServices; namespace Python.Test { @@ -83,7 +85,7 @@ public Type[] TestNullArrayConversion(Type[] v) public static string[] TestStringParamsArg(params string[] args) { - return args; + return args.Concat(new []{"tail"}).ToArray(); } public static object[] TestObjectParamsArg(params object[] args) @@ -651,6 +653,70 @@ public static string Casesensitive() { return "Casesensitive"; } + + public static string DefaultParams(int a=0, int b=0, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalParams([Optional]int a, [Optional]int b, [Optional]int c, [Optional] int d) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static bool OptionalParams_TestMissing([Optional]object a) + { + return a == Type.Missing; + } + + public static bool OptionalParams_TestReferenceType([Optional]string a) + { + return a == null; + } + + public static string OptionalAndDefaultParams([Optional]int a, [Optional]int b, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalAndDefaultParams2([Optional]int a, [Optional]int b, [Optional, DefaultParameterValue(1)]int c, int d = 2) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string DefaultParamsWithOverloading(int a = 2, int b = 1) + { + return $"{a}{b}"; + } + + public static string DefaultParamsWithOverloading(string a = "a", string b = "b") + { + return $"{a}{b}X"; + } + + public static string DefaultParamsWithOverloading(int a = 0, int b = 1, int c = 2) + { + return $"{a}{b}{c}XX"; + } + + public static string DefaultParamsWithOverloading(int a = 5, int b = 6, int c = 7, int d = 8) + { + return $"{a}{b}{c}{d}XXX"; + } + + public static string ParamsArrayOverloaded(int i = 1) + { + return "without params-array"; + } + + public static string ParamsArrayOverloaded(int i, params int[] paramsArray) + { + return "with params-array"; + } + + public static void EncodingTestÅngström() + { + } } @@ -665,4 +731,30 @@ public string PublicMethod(string echo) return echo; } } + + public class MethodArityTest + { + public string Foo(int a) { return "Arity 1"; } + public string Foo(int a, int b) { return "Arity 2"; } + } +} + +namespace PlainOldNamespace +{ + public class PlainOldClass + { + public PlainOldClass() { } + + public PlainOldClass(int param) { } + + private readonly byte[] payload = new byte[(int)Math.Pow(2, 20)]; //1 MB + + public void NonGenericMethod() { } + + public void GenericMethod() { } + + public void OverloadedMethod() { } + + public void OverloadedMethod(int param) { } + } } diff --git a/src/testing/mp_lengthtest.cs b/src/testing/mp_lengthtest.cs new file mode 100644 index 000000000..a4f3e8c25 --- /dev/null +++ b/src/testing/mp_lengthtest.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Python.Test +{ + public class MpLengthCollectionTest : ICollection + { + private readonly List items; + + public MpLengthCollectionTest() + { + SyncRoot = new object(); + items = new List + { + 1, + 2, + 3 + }; + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public void CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthExplicitCollectionTest : ICollection + { + private readonly List items; + private readonly object syncRoot; + + public MpLengthExplicitCollectionTest() + { + syncRoot = new object(); + items = new List + { + 9, + 10 + }; + } + int ICollection.Count => items.Count; + + object ICollection.SyncRoot => syncRoot; + + bool ICollection.IsSynchronized => false; + + void ICollection.CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthGenericCollectionTest() { + SyncRoot = new object(); + items = new List(); + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public bool IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + public void Clear() + { + items.Clear(); + } + + public bool Contains(T item) + { + return items.Contains(item); + } + + public void CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + public bool Remove(T item) + { + return items.Remove(item); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + } + + public class MpLengthExplicitGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthExplicitGenericCollectionTest() + { + items = new List(); + } + + int ICollection.Count => items.Count; + + bool ICollection.IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + void ICollection.Clear() + { + items.Clear(); + } + + bool ICollection.Contains(T item) + { + return items.Contains(item); + } + + void ICollection.CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + bool ICollection.Remove(T item) + { + return items.Remove(item); + } + } +} diff --git a/src/testing/nonexportable.cs b/src/testing/nonexportable.cs new file mode 100644 index 000000000..a29c78589 --- /dev/null +++ b/src/testing/nonexportable.cs @@ -0,0 +1,8 @@ +namespace Python.Test +{ + using Python.Runtime; + + // this class should not be visible to Python + [PyExport(false)] + public class NonExportable { } +} diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 9817d865e..ab0b73368 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -89,13 +89,24 @@ public static string test_bar(IInterfaceTest x, string s, int i) } // test instances can be constructed in managed code - public static IInterfaceTest create_instance(Type t) + public static SubClassTest create_instance(Type t) + { + return (SubClassTest)t.GetConstructor(new Type[] { }).Invoke(new object[] { }); + } + + public static IInterfaceTest create_instance_interface(Type t) { return (IInterfaceTest)t.GetConstructor(new Type[] { }).Invoke(new object[] { }); } - // test instances pass through managed code unchanged - public static IInterfaceTest pass_through(IInterfaceTest s) + // test instances pass through managed code unchanged ... + public static SubClassTest pass_through(SubClassTest s) + { + return s; + } + + // ... but the return type is an interface type, objects get wrapped + public static IInterfaceTest pass_through_interface(IInterfaceTest s) { return s; } diff --git a/src/testing/threadtest.cs b/src/testing/threadtest.cs index 2825c3fef..9c76929b2 100644 --- a/src/testing/threadtest.cs +++ b/src/testing/threadtest.cs @@ -11,8 +11,8 @@ public class ThreadTest private static PyObject module; private static string testmod = - "import CLR\n" + - "from CLR.Python.Test import ThreadTest\n" + + "import clr\n" + + "from Python.Test import ThreadTest\n" + "\n" + "def echostring(value):\n" + " return value\n" + diff --git a/src/tests/_compat.py b/src/tests/_compat.py deleted file mode 100644 index 3751ca013..000000000 --- a/src/tests/_compat.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Python 2.7, 3.3+ compatibility module. - -Using Python 3 syntax to encourage upgrade unless otherwise noted. -""" - -import operator -import subprocess -import sys -import types - -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY3: - import _thread as thread # Using PY2 name - import pickle - from collections import UserList - - indexbytes = operator.getitem - input = input - - string_types = str, - binary_type = bytes - text_type = str - - DictProxyType = type(object.__dict__) - ClassType = type - - # No PY3 equivalents, use PY2 name - long = int - unichr = chr - unicode = str - - # from nowhere import Nothing - cmp = lambda a, b: (a > b) - (a < b) # No PY3 equivalent - map = map - range = range - zip = zip - -elif PY2: - import thread # Using PY2 name - import cPickle as pickle - from UserList import UserList - - indexbytes = lambda buf, i: ord(buf[i]) - input = raw_input - - string_types = str, unicode - bytes_type = str - text_type = unicode - - DictProxyType = types.DictProxyType - ClassType = types.ClassType - - # No PY3 equivalents, use PY2 name - long = long - unichr = unichr - unicode = unicode - - from itertools import izip, imap - cmp = cmp - map = imap - range = xrange - zip = izip - - -def check_output(*args, **kwargs): - """Check output wrapper for PY2/PY3 compatibility""" - output = subprocess.check_output(*args, **kwargs) - if PY2: - return output - return output.decode("ascii") diff --git a/src/tests/conftest.py b/src/tests/conftest.py index a93326c51..17085e3ef 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -10,13 +10,21 @@ import sysconfig import pytest -import clr # Add path for `Python.Test` cwd = os.path.dirname(__file__) fixtures_path = os.path.join(cwd, "fixtures") + +BUILD_TEST = True +if BUILD_TEST: + from subprocess import check_call + test_proj_path = os.path.join(cwd, "..", "testing") + check_call(["dotnet", "build", test_proj_path, "-o", fixtures_path]) + sys.path.append(fixtures_path) +import clr + # Add References for tests clr.AddReference("Python.Test") clr.AddReference("System.Collections") diff --git a/src/tests/fixtures/netstandard2.0/.gitkeep b/src/tests/fixtures/netstandard2.0/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/src/tests/importtest.py b/src/tests/importtest.py new file mode 100644 index 000000000..fe93764e9 --- /dev/null +++ b/src/tests/importtest.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +import sys +try: + del sys.modules["System.IO"] +except KeyError: + pass + +assert "FileStream" not in globals() +import System.IO +from System.IO import * + +assert "FileStream" in globals() diff --git a/src/tests/leaktest.py b/src/tests/leaktest.py index 05b76e867..02133fece 100644 --- a/src/tests/leaktest.py +++ b/src/tests/leaktest.py @@ -10,7 +10,6 @@ import System -from ._compat import range from .utils import (CallableHandler, ClassMethodHandler, GenericHandler, HelloClass, StaticMethodHandler, VarCallableHandler, VariableArgsHandler, hello_func) diff --git a/src/tests/profile.py b/src/tests/profile.py index 4af3589e8..2113b1727 100644 --- a/src/tests/profile.py +++ b/src/tests/profile.py @@ -14,7 +14,6 @@ import time import runtests -from ._compat import range def main(): diff --git a/src/tests/runtests.py b/src/tests/runtests.py index 8011d05e6..9b90bcf6a 100644 --- a/src/tests/runtests.py +++ b/src/tests/runtests.py @@ -8,8 +8,6 @@ import sys import pytest -from ._compat import input - try: import System except ImportError: diff --git a/src/tests/stress.py b/src/tests/stress.py index c6fa8b7e3..f1f0fac6b 100644 --- a/src/tests/stress.py +++ b/src/tests/stress.py @@ -17,7 +17,7 @@ import threading import time -from ._compat import range, thread +import _thread as thread from .utils import dprint diff --git a/src/tests/stresstest.py b/src/tests/stresstest.py index 74b863bdc..b0dca9461 100644 --- a/src/tests/stresstest.py +++ b/src/tests/stresstest.py @@ -11,8 +11,6 @@ import unittest # import pdb -from ._compat import range - try: import System except ImportError: diff --git a/src/tests/test_array.py b/src/tests/test_array.py index 7ccadddff..9ab044b29 100644 --- a/src/tests/test_array.py +++ b/src/tests/test_array.py @@ -2,11 +2,12 @@ """Test support for managed arrays.""" +import clr import Python.Test as Test import System import pytest -from ._compat import PY2, UserList, long, range, unichr +from collections import UserList def test_public_array(): @@ -246,8 +247,8 @@ def test_char_array(): assert items[0] == 'a' assert items[4] == 'e' - max_ = unichr(65535) - min_ = unichr(0) + max_ = chr(65535) + min_ = chr(0) items[0] = max_ assert items[0] == max_ @@ -364,8 +365,8 @@ def test_int64_array(): assert items[0] == 0 assert items[4] == 4 - max_ = long(9223372036854775807) - min_ = long(-9223372036854775808) + max_ = 9223372036854775807 + min_ = -9223372036854775808 items[0] = max_ assert items[0] == max_ @@ -448,7 +449,7 @@ def test_uint32_array(): assert items[0] == 0 assert items[4] == 4 - max_ = long(4294967295) + max_ = 4294967295 min_ = 0 items[0] = max_ @@ -490,7 +491,7 @@ def test_uint64_array(): assert items[0] == 0 assert items[4] == 4 - max_ = long(18446744073709551615) + max_ = 18446744073709551615 min_ = 0 items[0] = max_ @@ -1143,6 +1144,8 @@ def test_boxed_value_type_mutation_result(): # to accidentally write code like the following which is not really # mutating value types in-place but changing boxed copies. + clr.AddReference('System.Drawing') + from System.Drawing import Point from System import Array @@ -1171,6 +1174,20 @@ def test_boxed_value_type_mutation_result(): assert items[i].X == i + 1 assert items[i].Y == i + 1 +def test_create_array_from_shape(): + from System import Array + + value = Array[int](3) + assert value[1] == 0 + assert value.Length == 3 + + value = Array[int](3, 4) + assert value[1, 1] == 0 + assert value.GetLength(0) == 3 + assert value.GetLength(1) == 4 + + with pytest.raises(ValueError): + Array[int](-1) def test_special_array_creation(): """Test using the Array[] syntax for creating arrays.""" @@ -1204,8 +1221,8 @@ def test_special_array_creation(): assert value.Length == 2 value = Array[System.Char]([0, 65535]) - assert value[0] == unichr(0) - assert value[1] == unichr(65535) + assert value[0] == chr(0) + assert value[1] == chr(65535) assert value.Length == 2 value = Array[System.Int16]([0, 32767]) @@ -1223,31 +1240,24 @@ def test_special_array_creation(): assert value[1] == 2147483647 assert value.Length == 2 - value = Array[System.Int64]([0, long(9223372036854775807)]) + value = Array[System.Int64]([0, 9223372036854775807]) assert value[0] == 0 - assert value[1] == long(9223372036854775807) + assert value[1] == 9223372036854775807 assert value.Length == 2 - # there's no explicit long type in python3, use System.Int64 instead - if PY2: - value = Array[long]([0, long(9223372036854775807)]) - assert value[0] == 0 - assert value[1] == long(9223372036854775807) - assert value.Length == 2 - value = Array[System.UInt16]([0, 65000]) assert value[0] == 0 assert value[1] == 65000 assert value.Length == 2 - value = Array[System.UInt32]([0, long(4294967295)]) + value = Array[System.UInt32]([0, 4294967295]) assert value[0] == 0 - assert value[1] == long(4294967295) + assert value[1] == 4294967295 assert value.Length == 2 - value = Array[System.UInt64]([0, long(18446744073709551615)]) + value = Array[System.UInt64]([0, 18446744073709551615]) assert value[0] == 0 - assert value[1] == long(18446744073709551615) + assert value[1] == 18446744073709551615 assert value.Length == 2 value = Array[System.Single]([0.0, 3.402823e38]) @@ -1295,9 +1305,10 @@ def test_special_array_creation(): assert value[1].__class__ == inst.__class__ assert value.Length == 2 + iface_class = ISayHello1(inst).__class__ value = Array[ISayHello1]([inst, inst]) - assert value[0].__class__ == inst.__class__ - assert value[1].__class__ == inst.__class__ + assert value[0].__class__ == iface_class + assert value[1].__class__ == iface_class assert value.Length == 2 inst = System.Exception("badness") @@ -1327,13 +1338,40 @@ def test_array_abuse(): with pytest.raises(TypeError): Test.PublicArrayTest.__getitem__(0, 0) - with pytest.raises(TypeError): + with pytest.raises(AttributeError): Test.PublicArrayTest.__setitem__(0, 0, 0) - with pytest.raises(TypeError): - desc = Test.PublicArrayTest.__dict__['__getitem__'] - desc(0, 0) + with pytest.raises(KeyError): + Test.PublicArrayTest.__dict__['__getitem__'] + + with pytest.raises(KeyError): + Test.PublicArrayTest.__dict__['__setitem__'] + + +def test_iterator_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + keys_iterator = iter(d.keys()) + arr = Array[String](keys_iterator) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + + +def test_dict_keys_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + d_keys = d.keys() + arr = Array[String](d_keys) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" - with pytest.raises(TypeError): - desc = Test.PublicArrayTest.__dict__['__setitem__'] - desc(0, 0, 0) diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 68773508b..4666631f7 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -3,18 +3,18 @@ """Test CLR class support.""" +import clr import Python.Test as Test import System import pytest -from ._compat import DictProxyType, range +from .utils import DictProxyType def test_basic_reference_type(): """Test usage of CLR defined reference types.""" assert System.String.Empty == "" - def test_basic_value_type(): """Test usage of CLR defined value types.""" assert System.Int32.MaxValue == 2147483647 @@ -29,7 +29,6 @@ def test_class_standard_attrs(): assert isinstance(ClassTest.__dict__, DictProxyType) assert len(ClassTest.__doc__) > 0 - def test_class_docstrings(): """Test standard class docstring generation""" from Python.Test import ClassTest @@ -58,6 +57,14 @@ def test_non_public_class(): with pytest.raises(AttributeError): _ = Test.InternalClass +def test_non_exported(): + """Test [PyExport(false)]""" + with pytest.raises(ImportError): + from Python.Test import NonExportable + + with pytest.raises(AttributeError): + _ = Test.NonExportable + def test_basic_subclass(): """Test basic subclass of a managed class.""" @@ -118,6 +125,9 @@ def __init__(self, v): def test_struct_construction(): """Test construction of structs.""" + + clr.AddReference('System.Drawing') + from System.Drawing import Point p = Point() @@ -166,6 +176,23 @@ def test_ienumerator_iteration(): for item in chars: assert item in 'test string' +def test_iterable(): + """Test what objects are Iterable""" + from collections.abc import Iterable + from Python.Test import ClassTest + + assert isinstance(System.String.Empty, Iterable) + assert isinstance(ClassTest.GetArrayList(), Iterable) + assert isinstance(ClassTest.GetEnumerator(), Iterable) + assert (not isinstance(ClassTest, Iterable)) + assert (not isinstance(ClassTest(), Iterable)) + + class ShouldBeIterable(ClassTest): + def __iter__(self): + return iter([]) + + assert isinstance(ShouldBeIterable(), Iterable) + def test_override_get_item(): """Test managed subclass overriding __getitem__.""" @@ -281,3 +308,24 @@ def PyCallback(self, self2): testobj.DoCallback() assert testobj.PyCallbackWasCalled assert testobj.SameReference + + +def test_method_inheritance(): + """Ensure that we call the overridden method instead of the one provided in + the base class.""" + + base = Test.BaseClass() + derived = Test.DerivedClass() + + assert base.IsBase() == True + assert derived.IsBase() == False + + +def test_callable(): + """Test that only delegate subtypes are callable""" + + def foo(): + pass + + assert callable(System.String("foo")) == False + assert callable(System.Action(foo)) == True diff --git a/src/tests/test_clrmethod.py b/src/tests/test_clrmethod.py new file mode 100644 index 000000000..0c63f9c59 --- /dev/null +++ b/src/tests/test_clrmethod.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +"""Test clrmethod and clrproperty support for calling methods and getting/setting python properties from CLR.""" + +import Python.Test as Test +import System +import pytest +import clr + +class ExampleClrClass(System.Object): + __namespace__ = "PyTest" + def __init__(self): + self._x = 3 + @clr.clrmethod(int, [int]) + def test(self, x): + return x*2 + + def get_X(self): + return self._x + def set_X(self, value): + self._x = value + X = clr.clrproperty(int, get_X, set_X) + + @clr.clrproperty(int) + def Y(self): + return self._x * 2 + +def test_set_and_get_property_from_py(): + """Test setting and getting clr-accessible properties from python.""" + t = ExampleClrClass() + assert t.X == 3 + assert t.Y == 3 * 2 + t.X = 4 + assert t.X == 4 + assert t.Y == 4 * 2 + +def test_set_and_get_property_from_clr(): + """Test setting and getting clr-accessible properties from the clr.""" + t = ExampleClrClass() + assert t.GetType().GetProperty("X").GetValue(t) == 3 + assert t.GetType().GetProperty("Y").GetValue(t) == 3 * 2 + t.GetType().GetProperty("X").SetValue(t, 4) + assert t.GetType().GetProperty("X").GetValue(t) == 4 + assert t.GetType().GetProperty("Y").GetValue(t) == 4 * 2 + + +def test_set_and_get_property_from_clr_and_py(): + """Test setting and getting clr-accessible properties alternatingly from the clr and from python.""" + t = ExampleClrClass() + assert t.GetType().GetProperty("X").GetValue(t) == 3 + assert t.GetType().GetProperty("Y").GetValue(t) == 3 * 2 + assert t.X == 3 + assert t.Y == 3 * 2 + t.GetType().GetProperty("X").SetValue(t, 4) + assert t.GetType().GetProperty("X").GetValue(t) == 4 + assert t.GetType().GetProperty("Y").GetValue(t) == 4 * 2 + assert t.X == 4 + assert t.Y == 4 * 2 + t.X = 5 + assert t.GetType().GetProperty("X").GetValue(t) == 5 + assert t.GetType().GetProperty("Y").GetValue(t) == 5 * 2 + assert t.X == 5 + assert t.Y == 5 * 2 + +def test_method_invocation_from_py(): + """Test calling a clr-accessible method from python.""" + t = ExampleClrClass() + assert t.test(41) == 41*2 + +def test_method_invocation_from_clr(): + """Test calling a clr-accessible method from the clr.""" + t = ExampleClrClass() + assert t.GetType().GetMethod("test").Invoke(t, [37]) == 37*2 diff --git a/src/tests/test_compat.py b/src/tests/test_compat.py deleted file mode 100644 index 81e7f8143..000000000 --- a/src/tests/test_compat.py +++ /dev/null @@ -1,243 +0,0 @@ -# -*- coding: utf-8 -*- -# TODO: Complete removal of methods below. Similar to test_module - -"""Backward-compatibility tests for deprecated features.""" - -import types - -import pytest - -from ._compat import ClassType, PY2, PY3, range -from .utils import is_clr_class, is_clr_module, is_clr_root_module - - -# Tests for old-style CLR-prefixed module naming. -def test_simple_import(): - """Test simple import.""" - import CLR - assert is_clr_root_module(CLR) - assert CLR.__name__ == 'clr' - - import sys - assert isinstance(sys, types.ModuleType) - assert sys.__name__ == 'sys' - - if PY3: - import http.client - assert isinstance(http.client, types.ModuleType) - assert http.client.__name__ == 'http.client' - - elif PY2: - import httplib - assert isinstance(httplib, types.ModuleType) - assert httplib.__name__ == 'httplib' - - -def test_simple_import_with_alias(): - """Test simple import with aliasing.""" - import CLR as myCLR - assert is_clr_root_module(myCLR) - assert myCLR.__name__ == 'clr' - - import sys as mySys - assert isinstance(mySys, types.ModuleType) - assert mySys.__name__ == 'sys' - - if PY3: - import http.client as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'http.client' - - elif PY2: - import httplib as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'httplib' - - -def test_dotted_name_import(): - """Test dotted-name import.""" - import CLR.System - assert is_clr_module(CLR.System) - assert CLR.System.__name__ == 'System' - - import System - assert is_clr_module(System) - assert System.__name__ == 'System' - - assert System is CLR.System - - import xml.dom - assert isinstance(xml.dom, types.ModuleType) - assert xml.dom.__name__ == 'xml.dom' - - -def test_dotted_name_import_with_alias(): - """Test dotted-name import with aliasing.""" - import CLR.System as myCLRSystem - assert is_clr_module(myCLRSystem) - assert myCLRSystem.__name__ == 'System' - - import System as mySystem - assert is_clr_module(mySystem) - assert mySystem.__name__ == 'System' - - assert mySystem is myCLRSystem - - import xml.dom as myDom - assert isinstance(myDom, types.ModuleType) - assert myDom.__name__ == 'xml.dom' - - -def test_simple_import_from(): - """Test simple 'import from'.""" - from CLR import System - assert is_clr_module(System) - assert System.__name__ == 'System' - - from xml import dom - assert isinstance(dom, types.ModuleType) - assert dom.__name__ == 'xml.dom' - - -def test_simple_import_from_with_alias(): - """Test simple 'import from' with aliasing.""" - from CLR import System as mySystem - assert is_clr_module(mySystem) - assert mySystem.__name__ == 'System' - - from xml import dom as myDom - assert isinstance(myDom, types.ModuleType) - assert myDom.__name__ == 'xml.dom' - - -def test_dotted_name_import_from(): - """Test dotted-name 'import from'.""" - from CLR.System import Xml - assert is_clr_module(Xml) - assert Xml.__name__ == 'System.Xml' - - from CLR.System.Xml import XmlDocument - assert is_clr_class(XmlDocument) - assert XmlDocument.__name__ == 'XmlDocument' - - from xml.dom import pulldom - assert isinstance(pulldom, types.ModuleType) - assert pulldom.__name__ == 'xml.dom.pulldom' - - from xml.dom.pulldom import PullDOM - assert isinstance(PullDOM, ClassType) - assert PullDOM.__name__ == 'PullDOM' - - -def test_dotted_name_import_from_with_alias(): - """Test dotted-name 'import from' with aliasing.""" - from CLR.System import Xml as myXml - assert is_clr_module(myXml) - assert myXml.__name__ == 'System.Xml' - - from CLR.System.Xml import XmlDocument as myXmlDocument - assert is_clr_class(myXmlDocument) - assert myXmlDocument.__name__ == 'XmlDocument' - - from xml.dom import pulldom as myPulldom - assert isinstance(myPulldom, types.ModuleType) - assert myPulldom.__name__ == 'xml.dom.pulldom' - - from xml.dom.pulldom import PullDOM as myPullDOM - assert isinstance(myPullDOM, ClassType) - assert myPullDOM.__name__ == 'PullDOM' - - -def test_from_module_import_star(): - """Test from module import * behavior.""" - count = len(locals().keys()) - m = __import__('CLR.System.Management', globals(), locals(), ['*']) - assert m.__name__ == 'System.Management' - assert is_clr_module(m) - assert len(locals().keys()) > count + 1 - - m2 = __import__('System.Management', globals(), locals(), ['*']) - assert m2.__name__ == 'System.Management' - assert is_clr_module(m2) - assert len(locals().keys()) > count + 1 - - assert m is m2 - - -def test_explicit_assembly_load(): - """Test explicit assembly loading using standard CLR tools.""" - from CLR.System.Reflection import Assembly - from CLR import System - import sys - - assembly = Assembly.LoadWithPartialName('System.Data') - assert assembly is not None - - import CLR.System.Data - assert 'System.Data' in sys.modules - - assembly = Assembly.LoadWithPartialName('SpamSpamSpamSpamEggsAndSpam') - assert assembly is None - - -def test_implicit_load_already_valid_namespace(): - """Test implicit assembly load over an already valid namespace.""" - # In this case, the mscorlib assembly (loaded by default) defines - # a number of types in the System namespace. There is also a System - # assembly, which is _not_ loaded by default, which also contains - # types in the System namespace. The desired behavior is for the - # Python runtime to "do the right thing", allowing types from both - # assemblies to be found in the CLR.System module implicitly. - import CLR.System - assert is_clr_class(CLR.System.UriBuilder) - - -def test_import_non_existant_module(): - """Test import failure for a non-existent module.""" - with pytest.raises(ImportError): - import System.SpamSpamSpam - - with pytest.raises(ImportError): - import CLR.System.SpamSpamSpam - - -def test_lookup_no_namespace_type(): - """Test lookup of types without a qualified namespace.""" - import CLR.Python.Test - import CLR - assert is_clr_class(CLR.NoNamespaceType) - - -def test_module_lookup_recursion(): - """Test for recursive lookup handling.""" - with pytest.raises(ImportError): - from CLR import CLR - - with pytest.raises(AttributeError): - import CLR - _ = CLR.CLR - - -def test_module_get_attr(): - """Test module getattr behavior.""" - import CLR.System as System - - int_type = System.Int32 - assert is_clr_class(int_type) - - module = System.Xml - assert is_clr_module(module) - - with pytest.raises(AttributeError): - _ = System.Spam - - with pytest.raises(TypeError): - _ = getattr(System, 1) - - -def test_multiple_imports(): - # import CLR did raise a Seg Fault once - # test if the Exceptions.warn() method still causes it - for _ in range(100): - import CLR - _ = CLR diff --git a/src/tests/test_constructors.py b/src/tests/test_constructors.py index 5e5489630..c305377f3 100644 --- a/src/tests/test_constructors.py +++ b/src/tests/test_constructors.py @@ -44,3 +44,12 @@ class Sub(System.Exception): instance = Sub() ob = SubclassConstructorTest(instance) assert isinstance(ob.value, System.Exception) + + +def test_multiple_constructor(): + from Python.Test import MultipleConstructorsTest + import System + + # Test parameterless + ob = MultipleConstructorsTest() + assert ob.value == "" diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index ac263ef5d..313274647 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- - """Test CLR <-> Python type conversions.""" -import System +import operator import pytest -from Python.Test import ConversionTest -from ._compat import indexbytes, long, unichr +import System +from Python.Test import ConversionTest, UnicodeString +from Python.Runtime import PyObjectConversions +from Python.Runtime.Codecs import RawProxyEncoder def test_bool_conversion(): @@ -142,8 +142,8 @@ def test_byte_conversion(): def test_char_conversion(): """Test char conversion.""" - assert System.Char.MaxValue == unichr(65535) - assert System.Char.MinValue == unichr(0) + assert System.Char.MaxValue == chr(65535) + assert System.Char.MinValue == chr(0) ob = ConversionTest() assert ob.CharField == u'A' @@ -247,23 +247,23 @@ def test_int32_conversion(): def test_int64_conversion(): """Test int64 conversion.""" - assert System.Int64.MaxValue == long(9223372036854775807) - assert System.Int64.MinValue == long(-9223372036854775808) + assert System.Int64.MaxValue == 9223372036854775807 + assert System.Int64.MinValue == -9223372036854775808 ob = ConversionTest() assert ob.Int64Field == 0 - ob.Int64Field = long(9223372036854775807) - assert ob.Int64Field == long(9223372036854775807) + ob.Int64Field = 9223372036854775807 + assert ob.Int64Field == 9223372036854775807 - ob.Int64Field = long(-9223372036854775808) - assert ob.Int64Field == long(-9223372036854775808) + ob.Int64Field = -9223372036854775808 + assert ob.Int64Field == -9223372036854775808 - ob.Int64Field = System.Int64(long(9223372036854775807)) - assert ob.Int64Field == long(9223372036854775807) + ob.Int64Field = System.Int64(9223372036854775807) + assert ob.Int64Field == 9223372036854775807 - ob.Int64Field = System.Int64(long(-9223372036854775808)) - assert ob.Int64Field == long(-9223372036854775808) + ob.Int64Field = System.Int64(-9223372036854775808) + assert ob.Int64Field == -9223372036854775808 with pytest.raises(TypeError): ConversionTest().Int64Field = "spam" @@ -272,16 +272,16 @@ def test_int64_conversion(): ConversionTest().Int64Field = None with pytest.raises(OverflowError): - ConversionTest().Int64Field = long(9223372036854775808) + ConversionTest().Int64Field = 9223372036854775808 with pytest.raises(OverflowError): - ConversionTest().Int64Field = long(-9223372036854775809) + ConversionTest().Int64Field = -9223372036854775809 with pytest.raises(OverflowError): - _ = System.Int64(long(9223372036854775808)) + _ = System.Int64(9223372036854775808) with pytest.raises(OverflowError): - _ = System.Int64(long(-9223372036854775809)) + _ = System.Int64(-9223372036854775809) def test_uint16_conversion(): @@ -325,38 +325,38 @@ def test_uint16_conversion(): def test_uint32_conversion(): """Test uint32 conversion.""" - assert System.UInt32.MaxValue == long(4294967295) + assert System.UInt32.MaxValue == 4294967295 assert System.UInt32.MinValue == 0 ob = ConversionTest() assert ob.UInt32Field == 0 - ob.UInt32Field = long(4294967295) - assert ob.UInt32Field == long(4294967295) + ob.UInt32Field = 4294967295 + assert ob.UInt32Field == 4294967295 ob.UInt32Field = -0 assert ob.UInt32Field == 0 - ob.UInt32Field = System.UInt32(long(4294967295)) - assert ob.UInt32Field == long(4294967295) + ob.UInt32Field = System.UInt32(4294967295) + assert ob.UInt32Field == 4294967295 ob.UInt32Field = System.UInt32(0) assert ob.UInt32Field == 0 - with pytest.raises(TypeError): + with pytest.raises(ValueError): ConversionTest().UInt32Field = "spam" with pytest.raises(TypeError): ConversionTest().UInt32Field = None with pytest.raises(OverflowError): - ConversionTest().UInt32Field = long(4294967296) + ConversionTest().UInt32Field = 4294967296 with pytest.raises(OverflowError): ConversionTest().UInt32Field = -1 with pytest.raises(OverflowError): - _ = System.UInt32(long(4294967296)) + _ = System.UInt32(4294967296) with pytest.raises(OverflowError): _ = System.UInt32(-1) @@ -364,38 +364,38 @@ def test_uint32_conversion(): def test_uint64_conversion(): """Test uint64 conversion.""" - assert System.UInt64.MaxValue == long(18446744073709551615) + assert System.UInt64.MaxValue == 18446744073709551615 assert System.UInt64.MinValue == 0 ob = ConversionTest() assert ob.UInt64Field == 0 - ob.UInt64Field = long(18446744073709551615) - assert ob.UInt64Field == long(18446744073709551615) + ob.UInt64Field = 18446744073709551615 + assert ob.UInt64Field == 18446744073709551615 ob.UInt64Field = -0 assert ob.UInt64Field == 0 - ob.UInt64Field = System.UInt64(long(18446744073709551615)) - assert ob.UInt64Field == long(18446744073709551615) + ob.UInt64Field = System.UInt64(18446744073709551615) + assert ob.UInt64Field == 18446744073709551615 ob.UInt64Field = System.UInt64(0) assert ob.UInt64Field == 0 - with pytest.raises(TypeError): + with pytest.raises(ValueError): ConversionTest().UInt64Field = "spam" with pytest.raises(TypeError): ConversionTest().UInt64Field = None with pytest.raises(OverflowError): - ConversionTest().UInt64Field = long(18446744073709551616) + ConversionTest().UInt64Field = 18446744073709551616 with pytest.raises(OverflowError): ConversionTest().UInt64Field = -1 with pytest.raises(OverflowError): - _ = System.UInt64(long(18446744073709551616)) + _ = System.UInt64((18446744073709551616)) with pytest.raises(OverflowError): _ = System.UInt64(-1) @@ -466,17 +466,6 @@ def test_double_conversion(): with pytest.raises(TypeError): ConversionTest().DoubleField = None - with pytest.raises(OverflowError): - ConversionTest().DoubleField = 1.7976931348623159e308 - - with pytest.raises(OverflowError): - ConversionTest().DoubleField = -1.7976931348623159e308 - - with pytest.raises(OverflowError): - _ = System.Double(1.7976931348623159e308) - - with pytest.raises(OverflowError): - _ = System.Double(-1.7976931348623159e308) def test_decimal_conversion(): @@ -486,7 +475,7 @@ def test_decimal_conversion(): max_d = Decimal.Parse("79228162514264337593543950335") min_d = Decimal.Parse("-79228162514264337593543950335") - assert Decimal.ToInt64(Decimal(10)) == long(10) + assert Decimal.ToInt64(Decimal(10)) == 10 ob = ConversionTest() assert ob.DecimalField == Decimal(0) @@ -547,6 +536,12 @@ def test_string_conversion(): with pytest.raises(TypeError): ConversionTest().StringField = 1 + world = UnicodeString() + test_unicode_str = u"안녕" + assert test_unicode_str == str(world.value) + assert test_unicode_str == str(world.GetString()) + assert test_unicode_str == str(world) + def test_interface_conversion(): """Test interface conversion.""" @@ -597,11 +592,10 @@ def test_object_conversion(): # need to test subclass here - with pytest.raises(TypeError): - class Foo(object): - pass - ob = ConversionTest() - ob.ObjectField = Foo + class Foo(object): + pass + ob.ObjectField = Foo + assert ob.ObjectField == Foo def test_enum_conversion(): @@ -641,6 +635,8 @@ def test_enum_conversion(): def test_null_conversion(): """Test null conversion.""" + import System + ob = ConversionTest() ob.StringField = None @@ -652,6 +648,10 @@ def test_null_conversion(): ob.SpamField = None assert ob.SpamField is None + pi = 22/7 + assert ob.Echo[System.Double](pi) == pi + assert ob.Echo[System.DateTime](None) is None + # Primitive types and enums should not be set to null. with pytest.raises(TypeError): @@ -677,7 +677,7 @@ def test_byte_array_conversion(): ob.ByteArrayField = value array = ob.ByteArrayField for i, _ in enumerate(value): - assert array[i] == indexbytes(value, i) + assert array[i] == operator.getitem(value, i) def test_sbyte_array_conversion(): @@ -696,4 +696,20 @@ def test_sbyte_array_conversion(): ob.SByteArrayField = value array = ob.SByteArrayField for i, _ in enumerate(value): - assert array[i] == indexbytes(value, i) + assert array[i] == operator.getitem(value, i) + +def test_codecs(): + """Test codec registration from Python""" + class ListAsRawEncoder(RawProxyEncoder): + __namespace__ = "Python.Test" + def CanEncode(self, clr_type): + return clr_type.Name == "List`1" and clr_type.Namespace == "System.Collections.Generic" + + list_raw_encoder = ListAsRawEncoder() + PyObjectConversions.RegisterEncoder(list_raw_encoder) + + ob = ConversionTest() + + l = ob.ListField + l.Add(42) + assert ob.ListField.Count == 1 diff --git a/src/tests/test_delegate.py b/src/tests/test_delegate.py index 33aca43b3..909fd0f05 100644 --- a/src/tests/test_delegate.py +++ b/src/tests/test_delegate.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# TODO: Add test for ObjectDelegate """Test CLR delegate support.""" @@ -8,8 +7,7 @@ import pytest from Python.Test import DelegateTest, StringDelegate -from ._compat import DictProxyType -from .utils import HelloClass, hello_func, MultipleHandler +from .utils import HelloClass, hello_func, MultipleHandler, DictProxyType def test_delegate_standard_attrs(): @@ -258,6 +256,26 @@ def always_so_negative(): assert not d() assert not ob.CallBoolDelegate(d) +def test_object_delegate(): + """Test object delegate.""" + from Python.Test import ObjectDelegate + + def create_object(): + return DelegateTest() + + d = ObjectDelegate(create_object) + ob = DelegateTest() + ob.CallObjectDelegate(d) + +def test_invalid_object_delegate(): + """Test invalid object delegate with mismatched return type.""" + from Python.Test import ObjectDelegate + + d = ObjectDelegate(hello_func) + ob = DelegateTest() + with pytest.raises(TypeError): + ob.CallObjectDelegate(d) + # test async delegates # test multicast delegates diff --git a/src/tests/test_enum.py b/src/tests/test_enum.py index b31ce4ec5..27fe7e9ef 100644 --- a/src/tests/test_enum.py +++ b/src/tests/test_enum.py @@ -5,7 +5,7 @@ import pytest import Python.Test as Test -from ._compat import DictProxyType, long +from .utils import DictProxyType def test_enum_standard_attrs(): @@ -68,23 +68,23 @@ def test_int_enum(): def test_uint_enum(): """Test uint enum.""" - assert Test.UIntEnum.Zero == long(0) - assert Test.UIntEnum.One == long(1) - assert Test.UIntEnum.Two == long(2) + assert Test.UIntEnum.Zero == 0 + assert Test.UIntEnum.One == 1 + assert Test.UIntEnum.Two == 2 def test_long_enum(): """Test long enum.""" - assert Test.LongEnum.Zero == long(0) - assert Test.LongEnum.One == long(1) - assert Test.LongEnum.Two == long(2) + assert Test.LongEnum.Zero == 0 + assert Test.LongEnum.One == 1 + assert Test.LongEnum.Two == 2 def test_ulong_enum(): """Test ulong enum.""" - assert Test.ULongEnum.Zero == long(0) - assert Test.ULongEnum.One == long(1) - assert Test.ULongEnum.Two == long(2) + assert Test.ULongEnum.Zero == 0 + assert Test.ULongEnum.One == 1 + assert Test.ULongEnum.Two == 2 def test_instantiate_enum_fails(): diff --git a/src/tests/test_event.py b/src/tests/test_event.py index 624b83d44..e9c0ffd8a 100644 --- a/src/tests/test_event.py +++ b/src/tests/test_event.py @@ -5,7 +5,6 @@ import pytest from Python.Test import EventTest, EventArgsTest -from ._compat import range from .utils import (CallableHandler, ClassMethodHandler, GenericHandler, MultipleHandler, StaticMethodHandler, VarCallableHandler, VariableArgsHandler) diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index f47209f6d..7fafeebcb 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -6,8 +6,7 @@ import System import pytest - -from ._compat import PY2, PY3, pickle, text_type +import pickle def test_unified_exception_semantics(): @@ -270,12 +269,6 @@ def test_str_of_exception(): with pytest.raises(FormatException) as cm: Convert.ToDateTime('this will fail') - e = cm.value - # fix for international installation - msg = text_type(e).encode("utf8") - fnd = text_type('System.Convert.ToDateTime').encode("utf8") - assert msg.find(fnd) > -1, msg - def test_python_compat_of_managed_exceptions(): """Test managed exceptions compatible with Python's implementation""" @@ -284,14 +277,11 @@ def test_python_compat_of_managed_exceptions(): e = OverflowException(msg) assert str(e) == msg - assert text_type(e) == msg assert e.args == (msg,) assert isinstance(e.args, tuple) - if PY3: - assert repr(e) == "OverflowException('Simple message',)" - elif PY2: - assert repr(e) == "OverflowException(u'Simple message',)" + strexp = "OverflowException('Simple message" + assert repr(e)[:len(strexp)] == strexp def test_exception_is_instance_of_system_object(): @@ -301,7 +291,7 @@ def test_exception_is_instance_of_system_object(): # classes, we wrap managed exceptions in a general-purpose old-style # class that delegates to the wrapped object. This makes _almost_ # everything work as expected, except that an isinstance check against - # CLR.System.Object will fail for a managed exception (because a new + # System.Object will fail for a managed exception (because a new # style class cannot appear in the __bases__ of an old-style class # without causing a crash in the CPython interpreter). This test is # here mainly to remind me to update the caveat in the documentation @@ -329,7 +319,6 @@ def test_pickling_exceptions(): assert exc.args == loaded.args -@pytest.mark.skipif(PY2, reason="__cause__ isn't implemented in PY2") def test_chained_exceptions(): from Python.Test import ExceptionTest @@ -345,3 +334,44 @@ def test_chained_exceptions(): assert exc.Message == msg assert exc.__cause__ == exc.InnerException exc = exc.__cause__ + +def test_iteration_exception(): + from Python.Test import ExceptionTest + from System import OverflowException + + exception = OverflowException("error") + + val = ExceptionTest.ThrowExceptionInIterator(exception).__iter__() + assert next(val) == 1 + assert next(val) == 2 + with pytest.raises(OverflowException) as cm: + next(val) + + exc = cm.value + + assert exc == exception + + # after exception is thrown iterator is no longer valid + with pytest.raises(StopIteration): + next(val) + + +def test_iteration_innerexception(): + from Python.Test import ExceptionTest + from System import OverflowException + + exception = System.Exception("message", OverflowException("error")) + + val = ExceptionTest.ThrowExceptionInIterator(exception).__iter__() + assert next(val) == 1 + assert next(val) == 2 + with pytest.raises(OverflowException) as cm: + next(val) + + exc = cm.value + + assert exc == exception.InnerException + + # after exception is thrown iterator is no longer valid + with pytest.raises(StopIteration): + next(val) diff --git a/src/tests/test_generic.py b/src/tests/test_generic.py index 69cd4ee7f..c865ab32f 100644 --- a/src/tests/test_generic.py +++ b/src/tests/test_generic.py @@ -7,8 +7,6 @@ import System import pytest -from ._compat import PY2, long, unicode, unichr, zip - def assert_generic_wrapper_by_type(ptype, value): """Test Helper""" @@ -137,15 +135,15 @@ def test_python_type_aliasing(): dict_.Add(1, 1) assert dict_[1] == 1 - dict_ = Dictionary[long, long]() + dict_ = Dictionary[int, int]() assert dict_.Count == 0 - dict_.Add(long(1), long(1)) - assert dict_[long(1)] == long(1) + dict_.Add(1, 1) + assert dict_[1] == 1 dict_ = Dictionary[System.Int64, System.Int64]() assert dict_.Count == 0 - dict_.Add(long(1), long(1)) - assert dict_[long(1)] == long(1) + dict_.Add(1, 1) + assert dict_[1] == 1 dict_ = Dictionary[float, float]() assert dict_.Count == 0 @@ -257,19 +255,15 @@ def test_generic_type_binding(): assert_generic_wrapper_by_type(System.Int16, 32767) assert_generic_wrapper_by_type(System.Int32, 2147483647) assert_generic_wrapper_by_type(int, 2147483647) - assert_generic_wrapper_by_type(System.Int64, long(9223372036854775807)) - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - assert_generic_wrapper_by_type(long, long(9223372036854775807)) + assert_generic_wrapper_by_type(System.Int64, 9223372036854775807) assert_generic_wrapper_by_type(System.UInt16, 65000) - assert_generic_wrapper_by_type(System.UInt32, long(4294967295)) - assert_generic_wrapper_by_type(System.UInt64, long(18446744073709551615)) + assert_generic_wrapper_by_type(System.UInt32, 4294967295) + assert_generic_wrapper_by_type(System.UInt64, 18446744073709551615) assert_generic_wrapper_by_type(System.Single, 3.402823e38) assert_generic_wrapper_by_type(System.Double, 1.7976931348623157e308) assert_generic_wrapper_by_type(float, 1.7976931348623157e308) assert_generic_wrapper_by_type(System.Decimal, System.Decimal.One) assert_generic_wrapper_by_type(System.String, "test") - assert_generic_wrapper_by_type(unicode, "test") assert_generic_wrapper_by_type(str, "test") assert_generic_wrapper_by_type(ShortEnum, ShortEnum.Zero) assert_generic_wrapper_by_type(System.Object, InterfaceTest()) @@ -315,24 +309,16 @@ def test_generic_method_type_handling(): assert_generic_method_by_type(System.Int16, 32767) assert_generic_method_by_type(System.Int32, 2147483647) assert_generic_method_by_type(int, 2147483647) - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - assert_generic_method_by_type(System.Int64, long(9223372036854775807)) - assert_generic_method_by_type(long, long(9223372036854775807)) - assert_generic_method_by_type(System.UInt32, long(4294967295)) - assert_generic_method_by_type(System.Int64, long(1844674407370955161)) assert_generic_method_by_type(System.UInt16, 65000) assert_generic_method_by_type(System.Single, 3.402823e38) assert_generic_method_by_type(System.Double, 1.7976931348623157e308) assert_generic_method_by_type(float, 1.7976931348623157e308) assert_generic_method_by_type(System.Decimal, System.Decimal.One) assert_generic_method_by_type(System.String, "test") - assert_generic_method_by_type(unicode, "test") assert_generic_method_by_type(str, "test") assert_generic_method_by_type(ShortEnum, ShortEnum.Zero) assert_generic_method_by_type(System.Object, InterfaceTest()) assert_generic_method_by_type(InterfaceTest, InterfaceTest(), 1) - assert_generic_method_by_type(ISayHello1, InterfaceTest(), 1) def test_correct_overload_selection(): @@ -355,8 +341,6 @@ def test_correct_overload_selection(): assert Math.Max(atype(value1), atype(value2)) == Math.Max.__overloads__[atype, atype]( atype(value1), atype(value2)) - if PY2 and atype is Int64: - value2 = long(value2) assert Math.Max(atype(value1), value2) == Math.Max.__overloads__[atype, atype]( atype(value1), atype(value2)) @@ -481,7 +465,7 @@ def test_method_overload_selection_with_generic_types(): vtype = GenericWrapper[System.Char] input_ = vtype(65535) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == unichr(65535) + assert value.value == chr(65535) vtype = GenericWrapper[System.Int16] input_ = vtype(32767) @@ -499,16 +483,9 @@ def test_method_overload_selection_with_generic_types(): assert value.value == 2147483647 vtype = GenericWrapper[System.Int64] - input_ = vtype(long(9223372036854775807)) + input_ = vtype(9223372036854775807) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(9223372036854775807) - - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - vtype = GenericWrapper[long] - input_ = vtype(long(9223372036854775807)) - value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(9223372036854775807) + assert value.value == 9223372036854775807 vtype = GenericWrapper[System.UInt16] input_ = vtype(65000) @@ -516,14 +493,14 @@ def test_method_overload_selection_with_generic_types(): assert value.value == 65000 vtype = GenericWrapper[System.UInt32] - input_ = vtype(long(4294967295)) + input_ = vtype(4294967295) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(4294967295) + assert value.value == 4294967295 vtype = GenericWrapper[System.UInt64] - input_ = vtype(long(18446744073709551615)) + input_ = vtype(18446744073709551615) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(18446744073709551615) + assert value.value == 18446744073709551615 vtype = GenericWrapper[System.Single] input_ = vtype(3.402823e38) @@ -570,10 +547,11 @@ def test_method_overload_selection_with_generic_types(): value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value.value.__class__ == inst.__class__ + iface_class = ISayHello1(inst).__class__ vtype = GenericWrapper[ISayHello1] input_ = vtype(inst) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value.__class__ == inst.__class__ + assert value.value.__class__ == iface_class vtype = System.Array[GenericWrapper[int]] input_ = vtype([GenericWrapper[int](0), GenericWrapper[int](1)]) @@ -628,7 +606,7 @@ def test_overload_selection_with_arrays_of_generic_types(): vtype = System.Array[gtype] input_ = vtype([gtype(65535), gtype(65535)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == unichr(65535) + assert value[0].value == chr(65535) assert value.Length == 2 gtype = GenericWrapper[System.Int16] @@ -654,22 +632,12 @@ def test_overload_selection_with_arrays_of_generic_types(): gtype = GenericWrapper[System.Int64] vtype = System.Array[gtype] - input_ = vtype([gtype(long(9223372036854775807)), - gtype(long(9223372036854775807))]) + input_ = vtype([gtype(9223372036854775807), + gtype(9223372036854775807)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(9223372036854775807) + assert value[0].value == 9223372036854775807 assert value.Length == 2 - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - gtype = GenericWrapper[long] - vtype = System.Array[gtype] - input_ = vtype([gtype(long(9223372036854775807)), - gtype(long(9223372036854775807))]) - value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(9223372036854775807) - assert value.Length == 2 - gtype = GenericWrapper[System.UInt16] vtype = System.Array[gtype] input_ = vtype([gtype(65000), gtype(65000)]) @@ -679,17 +647,17 @@ def test_overload_selection_with_arrays_of_generic_types(): gtype = GenericWrapper[System.UInt32] vtype = System.Array[gtype] - input_ = vtype([gtype(long(4294967295)), gtype(long(4294967295))]) + input_ = vtype([gtype(4294967295), gtype(4294967295)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(4294967295) + assert value[0].value == 4294967295 assert value.Length == 2 gtype = GenericWrapper[System.UInt64] vtype = System.Array[gtype] - input_ = vtype([gtype(long(18446744073709551615)), - gtype(long(18446744073709551615))]) + input_ = vtype([gtype(18446744073709551615), + gtype(18446744073709551615)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(18446744073709551615) + assert value[0].value == 18446744073709551615 assert value.Length == 2 gtype = GenericWrapper[System.Single] @@ -758,11 +726,12 @@ def test_overload_selection_with_arrays_of_generic_types(): assert value[0].value.__class__ == inst.__class__ assert value.Length == 2 + iface_class = ISayHello1(inst).__class__ gtype = GenericWrapper[ISayHello1] vtype = System.Array[gtype] input_ = vtype([gtype(inst), gtype(inst)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value.__class__ == inst.__class__ + assert value[0].value.__class__ == iface_class assert value.Length == 2 @@ -776,3 +745,8 @@ def test_nested_generic_class(): """Check nested generic classes.""" # TODO NotImplemented pass + +def test_missing_generic_type(): + from System.Collections import IList + with pytest.raises(TypeError): + IList[bool] diff --git a/src/tests/test_import.py b/src/tests/test_import.py index 42cafc4df..25877be15 100644 --- a/src/tests/test_import.py +++ b/src/tests/test_import.py @@ -3,7 +3,7 @@ """Test the import statement.""" import pytest - +import sys def test_relative_missing_import(): """Test that a relative missing import doesn't crash. @@ -11,3 +11,12 @@ def test_relative_missing_import(): Relative import in the site-packages folder""" with pytest.raises(ImportError): from . import _missing_import + + +def test_import_all_on_second_time(): + """Test import all attributes after a normal import without '*'. + Due to import * only allowed at module level, the test body splited + to a module file.""" + from . import importtest + del sys.modules[importtest.__name__] + diff --git a/src/tests/test_indexer.py b/src/tests/test_indexer.py index 6f18550d9..b8a33fb46 100644 --- a/src/tests/test_indexer.py +++ b/src/tests/test_indexer.py @@ -5,8 +5,6 @@ import Python.Test as Test import pytest -from ._compat import long, unichr - def test_public_indexer(): """Test public indexers.""" @@ -44,7 +42,7 @@ def test_internal_indexer(): with pytest.raises(TypeError): Test.InternalIndexerTest.__getitem__(ob, 0) - with pytest.raises(TypeError): + with pytest.raises(AttributeError): ob.__getitem__(0) @@ -58,7 +56,7 @@ def test_private_indexer(): with pytest.raises(TypeError): Test.PrivateIndexerTest.__getitem__(ob, 0) - with pytest.raises(TypeError): + with pytest.raises(AttributeError): ob.__getitem__(0) @@ -131,8 +129,8 @@ def test_sbyte_indexer(): def test_char_indexer(): """Test char indexers.""" ob = Test.CharIndexerTest() - max_ = unichr(65535) - min_ = unichr(0) + max_ = chr(65535) + min_ = chr(0) assert ob[max_] is None @@ -200,8 +198,8 @@ def test_int32_indexer(): def test_int64_indexer(): """Test Int64 indexers.""" ob = Test.Int64IndexerTest() - max_ = long(9223372036854775807) - min_ = long(-9223372036854775808) + max_ = 9223372036854775807 + min_ = -9223372036854775808 assert ob[max_] is None @@ -246,7 +244,7 @@ def test_uint16_indexer(): def test_uint32_indexer(): """Test UInt32 indexers.""" ob = Test.UInt32IndexerTest() - max_ = long(4294967295) + max_ = 4294967295 min_ = 0 assert ob[max_] is None @@ -269,7 +267,7 @@ def test_uint32_indexer(): def test_uint64_indexer(): """Test UInt64 indexers.""" ob = Test.UInt64IndexerTest() - max_ = long(18446744073709551615) + max_ = 18446744073709551615 min_ = 0 assert ob[max_] is None @@ -435,16 +433,16 @@ def test_object_indexer(): ob[1] = "one" assert ob[1] == "one" - ob[long(1)] = "long" - assert ob[long(1)] == "long" + ob[1] = "long" + assert ob[1] == "long" - with pytest.raises(TypeError): - class Eggs(object): - pass + class Eggs(object): + pass - key = Eggs() - ob = Test.ObjectIndexerTest() - ob[key] = "wrong" + key = Eggs() + ob = Test.ObjectIndexerTest() + ob[key] = "eggs_key" + assert ob[key] == "eggs_key" def test_interface_indexer(): @@ -597,3 +595,54 @@ def test_indexer_abuse(): with pytest.raises(AttributeError): del ob.__setitem__ + + +def test_indexer_accessed_through_interface(): + """Test that indexers can be accessed through interfaces""" + from System.Collections.Generic import Dictionary, IDictionary + d = IDictionary[str, str](Dictionary[str, str]()) + d["one"] = "1" + assert d["one"] == "1" + + +def test_using_indexer_on_object_without_indexer(): + """Test using subscript syntax on an object an without indexer raises""" + from System import Object + o = Object() + with pytest.raises(TypeError): + o[0] + + with pytest.raises(TypeError): + o[0] = 1 + + +def test_inherited_indexer(): + """Test that inherited indexers are accessible""" + from Python.Test import PublicInheritedIndexerTest + from Python.Test import ProtectedInheritedIndexerTest + from Python.Test import PrivateInheritedIndexerTest + from Python.Test import InternalInheritedIndexerTest + + pub = PublicInheritedIndexerTest() + pub[0] = "zero" + assert pub[0] == "zero" + + def assert_no_indexer(obj): + with pytest.raises(TypeError): + obj[0] + with pytest.raises(TypeError): + obj[0] = "zero" + + assert_no_indexer(PrivateInheritedIndexerTest) + assert_no_indexer(ProtectedInheritedIndexerTest) + assert_no_indexer(InternalInheritedIndexerTest) + + +def test_inherited_indexer_interface(): + """Test that indexers inherited from other interfaces are accessible""" + from Python.Test import InterfaceInheritedIndexerTest, IInheritedIndexer + + impl = InterfaceInheritedIndexerTest() + ifc = IInheritedIndexer(impl) + ifc[0] = "zero" + assert ifc[0] == "zero" diff --git a/src/tests/test_interface.py b/src/tests/test_interface.py index 997f17264..130bd71c1 100644 --- a/src/tests/test_interface.py +++ b/src/tests/test_interface.py @@ -5,7 +5,7 @@ import Python.Test as Test import pytest -from ._compat import DictProxyType +from .utils import DictProxyType def test_interface_standard_attrs(): @@ -61,9 +61,89 @@ def test_explicit_cast_to_interface(): assert hasattr(i1, 'SayHello') assert i1.SayHello() == 'hello 1' assert not hasattr(i1, 'HelloProperty') + assert i1.__implementation__ == ob + assert i1.__raw_implementation__ == ob i2 = Test.ISayHello2(ob) assert type(i2).__name__ == 'ISayHello2' assert i2.SayHello() == 'hello 2' assert hasattr(i2, 'SayHello') assert not hasattr(i2, 'HelloProperty') + + +def test_interface_object_returned_through_method(): + """Test interface type is used if method return type is interface""" + from Python.Test import InterfaceTest + + ob = InterfaceTest() + hello1 = ob.GetISayHello1() + assert type(hello1).__name__ == 'ISayHello1' + assert hello1.__implementation__.__class__.__name__ == "InterfaceTest" + + assert hello1.SayHello() == 'hello 1' + + +def test_interface_object_returned_through_out_param(): + """Test interface type is used for out parameters of interface types""" + from Python.Test import InterfaceTest + + ob = InterfaceTest() + hello2 = ob.GetISayHello2(None) + assert type(hello2).__name__ == 'ISayHello2' + + assert hello2.SayHello() == 'hello 2' + + +def test_null_interface_object_returned(): + """Test None is used also for methods with interface return types""" + from Python.Test import InterfaceTest + + ob = InterfaceTest() + hello1, hello2 = ob.GetNoSayHello(None) + assert hello1 is None + assert hello2 is None + +def test_interface_array_returned(): + """Test interface type used for methods returning interface arrays""" + from Python.Test import InterfaceTest + + ob = InterfaceTest() + hellos = ob.GetISayHello1Array() + assert type(hellos[0]).__name__ == 'ISayHello1' + assert hellos[0].__implementation__.__class__.__name__ == "InterfaceTest" + +def test_implementation_access(): + """Test the __implementation__ and __raw_implementation__ properties""" + import System + clrVal = System.Int32(100) + i = System.IComparable(clrVal) + assert 100 == i.__implementation__ + assert clrVal == i.__raw_implementation__ + assert i.__implementation__ != i.__raw_implementation__ + + +def test_interface_collection_iteration(): + """Test interface type is used when iterating over interface collection""" + import System + from System.Collections.Generic import List + elem = System.IComparable(System.Int32(100)) + typed_list = List[System.IComparable]() + typed_list.Add(elem) + for e in typed_list: + assert type(e).__name__ == "IComparable" + + untyped_list = System.Collections.ArrayList() + untyped_list.Add(elem) + for e in untyped_list: + assert type(e).__name__ == "int" + + +def test_methods_of_Object_are_available(): + """Test calling methods inherited from Object""" + import System + clrVal = System.Int32(100) + i = System.IComparable(clrVal) + assert i.Equals(clrVal) + assert clrVal.GetHashCode() == i.GetHashCode() + assert clrVal.GetType() == i.GetType() + assert clrVal.ToString() == i.ToString() diff --git a/src/tests/test_method.py b/src/tests/test_method.py index ad182678d..18eb5af8e 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -6,8 +6,6 @@ import pytest from Python.Test import MethodTest -from ._compat import PY2, long, unichr - def test_instance_method_descriptor(): """Test instance method descriptor behavior.""" @@ -208,17 +206,20 @@ def test_null_array_conversion(): def test_string_params_args(): """Test use of string params.""" result = MethodTest.TestStringParamsArg('one', 'two', 'three') - assert result.Length == 3 - assert len(result) == 3, result + assert result.Length == 4 + assert len(result) == 4, result assert result[0] == 'one' assert result[1] == 'two' assert result[2] == 'three' + # ensures params string[] overload takes precedence over params object[] + assert result[3] == 'tail' result = MethodTest.TestStringParamsArg(['one', 'two', 'three']) - assert len(result) == 3 + assert len(result) == 4 assert result[0] == 'one' assert result[1] == 'two' assert result[2] == 'three' + assert result[3] == 'tail' def test_object_params_args(): @@ -257,6 +258,37 @@ def test_non_params_array_in_last_place(): result = MethodTest.TestNonParamsArrayInLastPlace(1, 2, 3) assert result +def test_params_methods_with_no_params(): + """Tests that passing no arguments to a params method + passes an empty array""" + result = MethodTest.TestValueParamsArg() + assert len(result) == 0 + + result = MethodTest.TestOneArgWithParams('Some String') + assert len(result) == 0 + + result = MethodTest.TestTwoArgWithParams('Some String', 'Some Other String') + assert len(result) == 0 + +def test_params_methods_with_non_lists(): + """Tests that passing single parameters to a params + method will convert into an array on the .NET side""" + result = MethodTest.TestOneArgWithParams('Some String', [1, 2, 3, 4]) + assert len(result) == 4 + + result = MethodTest.TestOneArgWithParams('Some String', 1, 2, 3, 4) + assert len(result) == 4 + + result = MethodTest.TestOneArgWithParams('Some String', [5]) + assert len(result) == 1 + + result = MethodTest.TestOneArgWithParams('Some String', 5) + assert len(result) == 1 + +def test_params_method_with_lists(): + """Tests passing multiple lists to a params object[] method""" + result = MethodTest.TestObjectParamsArg([],[]) + assert len(result) == 2 def test_string_out_params(): """Test use of string out-parameters.""" @@ -473,7 +505,7 @@ def test_explicit_overload_selection(): assert value == u'A' value = MethodTest.Overloaded.__overloads__[System.Char](65535) - assert value == unichr(65535) + assert value == chr(65535) value = MethodTest.Overloaded.__overloads__[System.Int16](32767) assert value == 32767 @@ -485,25 +517,22 @@ def test_explicit_overload_selection(): assert value == 2147483647 value = MethodTest.Overloaded.__overloads__[System.Int64]( - long(9223372036854775807)) - assert value == long(9223372036854775807) - - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - value = MethodTest.Overloaded.__overloads__[long]( - long(9223372036854775807)) - assert value == long(9223372036854775807) + 9223372036854775807 + ) + assert value == 9223372036854775807 value = MethodTest.Overloaded.__overloads__[System.UInt16](65000) assert value == 65000 value = MethodTest.Overloaded.__overloads__[System.UInt32]( - long(4294967295)) - assert value == long(4294967295) + 4294967295 + ) + assert value == 4294967295 value = MethodTest.Overloaded.__overloads__[System.UInt64]( - long(18446744073709551615)) - assert value == long(18446744073709551615) + 18446744073709551615 + ) + assert value == 18446744073709551615 value = MethodTest.Overloaded.__overloads__[System.Single](3.402823e38) assert value == 3.402823e38 @@ -535,8 +564,10 @@ def test_explicit_overload_selection(): value = MethodTest.Overloaded.__overloads__[InterfaceTest](inst) assert value.__class__ == inst.__class__ + iface_class = ISayHello1(InterfaceTest()).__class__ value = MethodTest.Overloaded.__overloads__[ISayHello1](inst) - assert value.__class__ == inst.__class__ + assert value.__class__ != inst.__class__ + assert value.__class__ == iface_class atype = Array[System.Object] value = MethodTest.Overloaded.__overloads__[str, int, atype]( @@ -590,8 +621,8 @@ def test_overload_selection_with_array_types(): vtype = Array[System.Char] input_ = vtype([0, 65535]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0] == unichr(0) - assert value[1] == unichr(65535) + assert value[0] == chr(0) + assert value[1] == chr(65535) vtype = Array[System.Int16] input_ = vtype([0, 32767]) @@ -612,18 +643,10 @@ def test_overload_selection_with_array_types(): assert value[1] == 2147483647 vtype = Array[System.Int64] - input_ = vtype([0, long(9223372036854775807)]) + input_ = vtype([0, 9223372036854775807]) value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value[0] == 0 - assert value[1] == long(9223372036854775807) - - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - vtype = Array[long] - input_ = vtype([0, long(9223372036854775807)]) - value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0] == 0 - assert value[1] == long(9223372036854775807) + assert value[1] == 9223372036854775807 vtype = Array[System.UInt16] input_ = vtype([0, 65000]) @@ -632,16 +655,16 @@ def test_overload_selection_with_array_types(): assert value[1] == 65000 vtype = Array[System.UInt32] - input_ = vtype([0, long(4294967295)]) + input_ = vtype([0, 4294967295]) value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value[0] == 0 - assert value[1] == long(4294967295) + assert value[1] == 4294967295 vtype = Array[System.UInt64] - input_ = vtype([0, long(18446744073709551615)]) + input_ = vtype([0, 18446744073709551615]) value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value[0] == 0 - assert value[1] == long(18446744073709551615) + assert value[1] == 18446744073709551615 vtype = Array[System.Single] input_ = vtype([0.0, 3.402823e38]) @@ -697,11 +720,12 @@ def test_overload_selection_with_array_types(): assert value[0].__class__ == inst.__class__ assert value[1].__class__ == inst.__class__ + iface_class = ISayHello1(inst).__class__ vtype = Array[ISayHello1] input_ = vtype([inst, inst]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].__class__ == inst.__class__ - assert value[1].__class__ == inst.__class__ + assert value[0].__class__ == iface_class + assert value[1].__class__ == iface_class def test_explicit_overload_selection_failure(): @@ -717,7 +741,7 @@ def test_explicit_overload_selection_failure(): _ = MethodTest.Overloaded.__overloads__[str, int, int]("", 1, 1) with pytest.raises(TypeError): - _ = MethodTest.Overloaded.__overloads__[int, long](1) + _ = MethodTest.Overloaded.__overloads__[int, int](1) def test_we_can_bind_to_encoding_get_string(): @@ -737,7 +761,7 @@ def test_we_can_bind_to_encoding_get_string(): read = 1 while read > 0: - read, _ = stream.Read(buff, 0, buff.Length) + read = stream.Read(buff, 0, buff.Length) temp = Encoding.UTF8.GetString(buff, 0, read) data.append(temp) @@ -777,6 +801,9 @@ def test_no_object_in_param(): res = MethodTest.TestOverloadedNoObject(5) assert res == "Got int" + res = MethodTest.TestOverloadedNoObject(i=7) + assert res == "Got int" + with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject("test") @@ -788,9 +815,15 @@ def test_object_in_param(): res = MethodTest.TestOverloadedObject(5) assert res == "Got int" + res = MethodTest.TestOverloadedObject(i=7) + assert res == "Got int" + res = MethodTest.TestOverloadedObject("test") assert res == "Got object" + res = MethodTest.TestOverloadedObject(o="test") + assert res == "Got object" + def test_object_in_multiparam(): """Test method with object multiparams behaves""" @@ -813,6 +846,42 @@ def test_object_in_multiparam(): res = MethodTest.TestOverloadedObjectTwo(7.24, 7.24) assert res == "Got object-object" + res = MethodTest.TestOverloadedObjectTwo(a=5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(a=5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo("foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo(a=7.24, b=7.24) + assert res == "Got object-object" + + res = MethodTest.TestOverloadedObjectTwo(7.24, b=7.24) + assert res == "Got object-object" + def test_object_in_multiparam_exception(): """Test method with object multiparams behaves""" @@ -832,3 +901,325 @@ def test_case_sensitive(): with pytest.raises(AttributeError): MethodTest.casesensitive() + +def test_getting_generic_method_binding_does_not_leak_ref_count(): + """Test that managed object is freed after calling generic method. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import sys + + refCount = sys.getrefcount(PlainOldClass().GenericMethod[str]) + assert refCount == 1 + +def test_getting_generic_method_binding_does_not_leak_memory(): + """Test that managed object is freed after calling generic method. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import psutil, os, gc, clr + + process = psutil.Process(os.getpid()) + processBytesBeforeCall = process.memory_info().rss + print("\n\nMemory consumption (bytes) at start of test: " + str(processBytesBeforeCall)) + + iterations = 500 + for i in range(iterations): + PlainOldClass().GenericMethod[str] + + gc.collect() + System.GC.Collect() + + processBytesAfterCall = process.memory_info().rss + print("Memory consumption (bytes) at end of test: " + str(processBytesAfterCall)) + processBytesDelta = processBytesAfterCall - processBytesBeforeCall + print("Memory delta: " + str(processBytesDelta)) + + bytesAllocatedPerIteration = pow(2, 20) # 1MB + bytesLeakedPerIteration = processBytesDelta / iterations + + # Allow 50% threshold - this shows the original issue is fixed, which leaks the full allocated bytes per iteration + failThresholdBytesLeakedPerIteration = bytesAllocatedPerIteration / 2 + + assert bytesLeakedPerIteration < failThresholdBytesLeakedPerIteration + +def test_getting_overloaded_method_binding_does_not_leak_ref_count(): + """Test that managed object is freed after calling overloaded method. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import sys + + refCount = sys.getrefcount(PlainOldClass().OverloadedMethod.Overloads[int]) + assert refCount == 1 + +def test_getting_overloaded_method_binding_does_not_leak_memory(): + """Test that managed object is freed after calling overloaded method. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import psutil, os, gc, clr + + process = psutil.Process(os.getpid()) + processBytesBeforeCall = process.memory_info().rss + print("\n\nMemory consumption (bytes) at start of test: " + str(processBytesBeforeCall)) + + iterations = 500 + for i in range(iterations): + PlainOldClass().OverloadedMethod.Overloads[int] + + gc.collect() + System.GC.Collect() + + processBytesAfterCall = process.memory_info().rss + print("Memory consumption (bytes) at end of test: " + str(processBytesAfterCall)) + processBytesDelta = processBytesAfterCall - processBytesBeforeCall + print("Memory delta: " + str(processBytesDelta)) + + bytesAllocatedPerIteration = pow(2, 20) # 1MB + bytesLeakedPerIteration = processBytesDelta / iterations + + # Allow 50% threshold - this shows the original issue is fixed, which leaks the full allocated bytes per iteration + failThresholdBytesLeakedPerIteration = bytesAllocatedPerIteration / 2 + + assert bytesLeakedPerIteration < failThresholdBytesLeakedPerIteration + +def test_getting_method_overloads_binding_does_not_leak_ref_count(): + """Test that managed object is freed after calling overloaded method. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import sys + + refCount = sys.getrefcount(PlainOldClass().OverloadedMethod.Overloads) + assert refCount == 1 + +def test_getting_method_overloads_binding_does_not_leak_memory(): + """Test that managed object is freed after calling overloaded method. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import psutil, os, gc, clr + + process = psutil.Process(os.getpid()) + processBytesBeforeCall = process.memory_info().rss + print("\n\nMemory consumption (bytes) at start of test: " + str(processBytesBeforeCall)) + + iterations = 500 + for i in range(iterations): + PlainOldClass().OverloadedMethod.Overloads + + gc.collect() + System.GC.Collect() + + processBytesAfterCall = process.memory_info().rss + print("Memory consumption (bytes) at end of test: " + str(processBytesAfterCall)) + processBytesDelta = processBytesAfterCall - processBytesBeforeCall + print("Memory delta: " + str(processBytesDelta)) + + bytesAllocatedPerIteration = pow(2, 20) # 1MB + bytesLeakedPerIteration = processBytesDelta / iterations + + # Allow 50% threshold - this shows the original issue is fixed, which leaks the full allocated bytes per iteration + failThresholdBytesLeakedPerIteration = bytesAllocatedPerIteration / 2 + + assert bytesLeakedPerIteration < failThresholdBytesLeakedPerIteration + +def test_getting_overloaded_constructor_binding_does_not_leak_ref_count(): + """Test that managed object is freed after calling overloaded constructor, constructorbinding.cs mp_subscript. Issue #691""" + + from PlainOldNamespace import PlainOldClass + + import sys + + # simple test + refCount = sys.getrefcount(PlainOldClass.Overloads[int]) + assert refCount == 1 + + +def test_default_params(): + # all positional parameters + res = MethodTest.DefaultParams(1,2,3,4) + assert res == "1234" + + res = MethodTest.DefaultParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.DefaultParams(1, 2) + assert res == "1200" + + res = MethodTest.DefaultParams(1) + assert res == "1000" + + res = MethodTest.DefaultParams(a=2) + assert res == "2000" + + res = MethodTest.DefaultParams(b=3) + assert res == "0300" + + res = MethodTest.DefaultParams(c=4) + assert res == "0040" + + res = MethodTest.DefaultParams(d=7) + assert res == "0007" + + res = MethodTest.DefaultParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.DefaultParams(1, d=7, c=3) + assert res == "1037" + + with pytest.raises(TypeError): + MethodTest.DefaultParams(1,2,3,4,5) + +def test_optional_params(): + res = MethodTest.OptionalParams(1, 2, 3, 4) + assert res == "1234" + + res = MethodTest.OptionalParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.OptionalParams(1, 2) + assert res == "1200" + + res = MethodTest.OptionalParams(1) + assert res == "1000" + + res = MethodTest.OptionalParams(a=2) + assert res == "2000" + + res = MethodTest.OptionalParams(b=3) + assert res == "0300" + + res = MethodTest.OptionalParams(c=4) + assert res == "0040" + + res = MethodTest.OptionalParams(d=7) + assert res == "0007" + + res = MethodTest.OptionalParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.OptionalParams(1, d=7, c=3) + assert res == "1037" + + res = MethodTest.OptionalParams_TestMissing() + assert res == True + + res = MethodTest.OptionalParams_TestMissing(None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a = None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a='hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType() + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(a=None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType('hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType(a='hi') + assert res == False + +def test_optional_and_default_params(): + + res = MethodTest.OptionalAndDefaultParams() + assert res == "0000" + + res = MethodTest.OptionalAndDefaultParams(1) + assert res == "1000" + + res = MethodTest.OptionalAndDefaultParams(1, c=4) + assert res == "1040" + + res = MethodTest.OptionalAndDefaultParams(b=4, c=7) + assert res == "0470" + + res = MethodTest.OptionalAndDefaultParams2() + assert res == "0012" + + res = MethodTest.OptionalAndDefaultParams2(a=1,b=2,c=3,d=4) + assert res == "1234" + + res = MethodTest.OptionalAndDefaultParams2(b=2, c=3) + assert res == "0232" + +def test_default_params_overloads(): + res = MethodTest.DefaultParamsWithOverloading(1, 2) + assert res == "12" + + res = MethodTest.DefaultParamsWithOverloading(b=5) + assert res == "25" + + res = MethodTest.DefaultParamsWithOverloading("d") + assert res == "dbX" + + res = MethodTest.DefaultParamsWithOverloading(b="c") + assert res == "acX" + + res = MethodTest.DefaultParamsWithOverloading(c=3) + assert res == "013XX" + + res = MethodTest.DefaultParamsWithOverloading(5, c=2) + assert res == "512XX" + + res = MethodTest.DefaultParamsWithOverloading(c=0, d=1) + assert res == "5601XXX" + + res = MethodTest.DefaultParamsWithOverloading(1, d=1) + assert res == "1671XXX" + +def test_default_params_overloads_ambiguous_call(): + with pytest.raises(TypeError): + MethodTest.DefaultParamsWithOverloading() + +def test_keyword_arg_method_resolution(): + from Python.Test import MethodArityTest + + ob = MethodArityTest() + assert ob.Foo(1, b=2) == "Arity 2" + +def test_params_array_overload(): + res = MethodTest.ParamsArrayOverloaded() + assert res == "without params-array" + + res = MethodTest.ParamsArrayOverloaded(1) + assert res == "without params-array" + + res = MethodTest.ParamsArrayOverloaded(i=1) + assert res == "without params-array" + + res = MethodTest.ParamsArrayOverloaded(1, 2) + assert res == "with params-array" + + res = MethodTest.ParamsArrayOverloaded(1, 2, 3) + assert res == "with params-array" + + res = MethodTest.ParamsArrayOverloaded(1, paramsArray=[]) + assert res == "with params-array" + + res = MethodTest.ParamsArrayOverloaded(1, i=1) + assert res == "with params-array" + + res = MethodTest.ParamsArrayOverloaded(1, 2, 3, i=1) + assert res == "with params-array" + + # These two cases are still incorrectly failing: + + # res = MethodTest.ParamsArrayOverloaded(1, 2, i=1) + # assert res == "with params-array" + + # res = MethodTest.ParamsArrayOverloaded(paramsArray=[], i=1) + # assert res == "with params-array" + +def test_method_encoding(): + MethodTest.EncodingTestÅngström() diff --git a/src/tests/test_module.py b/src/tests/test_module.py index 2255ea411..dcdb0248e 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -10,7 +10,6 @@ import pytest -from ._compat import ClassType, PY2, PY3, range from .utils import is_clr_class, is_clr_module, is_clr_root_module @@ -79,14 +78,9 @@ def test_simple_import(): assert isinstance(sys, types.ModuleType) assert sys.__name__ == 'sys' - if PY3: - import http.client as httplib - assert isinstance(httplib, types.ModuleType) - assert httplib.__name__ == 'http.client' - elif PY2: - import httplib - assert isinstance(httplib, types.ModuleType) - assert httplib.__name__ == 'httplib' + import http.client as httplib + assert isinstance(httplib, types.ModuleType) + assert httplib.__name__ == 'http.client' def test_simple_import_with_alias(): @@ -99,14 +93,9 @@ def test_simple_import_with_alias(): assert isinstance(mySys, types.ModuleType) assert mySys.__name__ == 'sys' - if PY3: - import http.client as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'http.client' - elif PY2: - import httplib as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'httplib' + import http.client as myHttplib + assert isinstance(myHttplib, types.ModuleType) + assert myHttplib.__name__ == 'http.client' def test_dotted_name_import(): @@ -178,7 +167,7 @@ def test_dotted_name_import_from(): assert pulldom.__name__ == 'xml.dom.pulldom' from xml.dom.pulldom import PullDOM - assert isinstance(PullDOM, ClassType) + assert isinstance(PullDOM, type) assert PullDOM.__name__ == 'PullDOM' @@ -197,12 +186,14 @@ def test_dotted_name_import_from_with_alias(): assert myPulldom.__name__ == 'xml.dom.pulldom' from xml.dom.pulldom import PullDOM as myPullDOM - assert isinstance(myPullDOM, ClassType) + assert isinstance(myPullDOM, type) assert myPullDOM.__name__ == 'PullDOM' def test_from_module_import_star(): """Test from module import * behavior.""" + clr.AddReference('System.Xml') + count = len(locals().keys()) m = __import__('System.Xml', globals(), locals(), ['*']) assert m.__name__ == 'System.Xml' @@ -212,16 +203,14 @@ def test_from_module_import_star(): def test_implicit_assembly_load(): """Test implicit assembly loading via import.""" - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") + with pytest.raises(ImportError): + # MS.Build should not have been added as a reference yet + # (and should exist for mono) - # should trigger a DeprecationWarning as Microsoft.Build hasn't - # been added as a reference yet (and should exist for mono) + # The implicit behavior has been disabled in 3.0 + # therefore this should fail import Microsoft.Build - assert len(w) == 1 - assert isinstance(w[0].message, DeprecationWarning) - with warnings.catch_warnings(record=True) as w: clr.AddReference("System.Windows.Forms") import System.Windows.Forms as Forms @@ -352,6 +341,26 @@ def test_clr_add_reference(): with pytest.raises(FileNotFoundException): AddReference("somethingtotallysilly") +def test_clr_get_clr_type(): + """Test clr.GetClrType().""" + from clr import GetClrType + import System + from System import IComparable + from System import ArgumentException + assert GetClrType(System.String).FullName == "System.String" + comparable = GetClrType(IComparable) + assert comparable.FullName == "System.IComparable" + assert comparable.IsInterface + assert GetClrType(int).FullName == "System.Int32" + assert GetClrType(str).FullName == "System.String" + assert GetClrType(float).FullName == "System.Double" + dblarr = System.Array[System.Double] + assert GetClrType(dblarr).FullName == "System.Double[]" + + with pytest.raises(TypeError): + GetClrType(1) + with pytest.raises(TypeError): + GetClrType("thiswillfail") def test_assembly_load_thread_safety(): from Python.Test import ModuleTest @@ -367,3 +376,9 @@ def test_assembly_load_thread_safety(): from System.Collections.Generic import Dictionary _ = Dictionary[Guid, DateTime]() ModuleTest.JoinThreads() + +def test_assembly_load_recursion_bug(): + """Test fix for recursion bug documented in #627""" + from System.Configuration import ConfigurationManager + content = dir(ConfigurationManager) + assert len(content) > 5, content diff --git a/src/tests/test_mp_length.py b/src/tests/test_mp_length.py new file mode 100644 index 000000000..e86fff288 --- /dev/null +++ b/src/tests/test_mp_length.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +"""Test __len__ for .NET classes implementing ICollection/ICollection.""" + +import System +import pytest +from Python.Test import MpLengthCollectionTest, MpLengthExplicitCollectionTest, MpLengthGenericCollectionTest, MpLengthExplicitGenericCollectionTest + +def test_simple___len__(): + """Test __len__ for simple ICollection implementers""" + import System + import System.Collections.Generic + l = System.Collections.Generic.List[int]() + assert len(l) == 0 + l.Add(5) + l.Add(6) + assert len(l) == 2 + + d = System.Collections.Generic.Dictionary[int, int]() + assert len(d) == 0 + d.Add(4, 5) + assert len(d) == 1 + + a = System.Array[int]([0,1,2,3]) + assert len(a) == 4 + +def test_custom_collection___len__(): + """Test __len__ for custom collection class""" + s = MpLengthCollectionTest() + assert len(s) == 3 + +def test_custom_collection_explicit___len__(): + """Test __len__ for custom collection class that explicitly implements ICollection""" + s = MpLengthExplicitCollectionTest() + assert len(s) == 2 + +def test_custom_generic_collection___len__(): + """Test __len__ for custom generic collection class""" + s = MpLengthGenericCollectionTest[int]() + s.Add(1) + s.Add(2) + assert len(s) == 2 + +def test_custom_generic_collection_explicit___len__(): + """Test __len__ for custom generic collection that explicity implements ICollection""" + s = MpLengthExplicitGenericCollectionTest[int]() + s.Add(1) + s.Add(10) + assert len(s) == 2 + +def test_len_through_interface_generic(): + """Test __len__ for ICollection""" + import System.Collections.Generic + l = System.Collections.Generic.List[int]() + coll = System.Collections.Generic.ICollection[int](l) + assert len(coll) == 0 + +def test_len_through_interface(): + """Test __len__ for ICollection""" + import System.Collections + l = System.Collections.ArrayList() + coll = System.Collections.ICollection(l) + assert len(coll) == 0 diff --git a/src/tests/test_repr.py b/src/tests/test_repr.py new file mode 100644 index 000000000..5131f5d88 --- /dev/null +++ b/src/tests/test_repr.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +"""Test __repr__ output""" + +import System +import pytest +from Python.Test import ReprTest + +def test_basic(): + """Test Point class which implements both ToString and __repr__ without inheritance""" + ob = ReprTest.Point(1,2) + # point implements ToString() and __repr__() + assert ob.__repr__() == "Point(1,2)" + assert str(ob) == "Python.Test.ReprTest+Point: X=1, Y=2" + +def test_system_string(): + """Test system string""" + ob = System.String("hello") + assert str(ob) == "hello" + assert " IInterfaceTest, + # causing a new wrapper object to be created. Hence id will differ. + x = FunctionsTest.pass_through_interface(ob) + assert id(x) != id(ob) -@pytest.mark.skip(reason="FIXME: test randomly pass/fails") def test_derived_class(): """Test python class derived from managed type""" - DerivedClass = derived_class_fixture() + DerivedClass = derived_class_fixture(test_derived_class.__name__) ob = DerivedClass() assert ob.foo() == "DerivedClass" assert ob.base_foo() == "foo" @@ -131,10 +128,42 @@ def test_derived_class(): assert id(x) == id(ob) -@pytest.mark.skip(reason="FIXME: test randomly pass/fails") +def test_derived_traceback(): + """Test python exception traceback in class derived from managed base""" + class DerivedClass(SubClassTest): + __namespace__ = "Python.Test.traceback" + + def foo(self): + print (xyzname) + return None + + import sys,traceback + ob = DerivedClass() + + # direct call + try: + ob.foo() + assert False + except: + e = sys.exc_info() + assert "xyzname" in str(e[1]) + location = traceback.extract_tb(e[2])[-1] + assert location[2] == "foo" + + # call through managed code + try: + FunctionsTest.test_foo(ob) + assert False + except: + e = sys.exc_info() + assert "xyzname" in str(e[1]) + location = traceback.extract_tb(e[2])[-1] + assert location[2] == "foo" + + def test_create_instance(): """Test derived instances can be created from managed code""" - DerivedClass = derived_class_fixture() + DerivedClass = derived_class_fixture(test_create_instance.__name__) ob = FunctionsTest.create_instance(DerivedClass) assert ob.foo() == "DerivedClass" assert FunctionsTest.test_foo(ob) == "DerivedClass" @@ -145,18 +174,17 @@ def test_create_instance(): x = FunctionsTest.pass_through(ob) assert id(x) == id(ob) - InterfaceTestClass = interface_test_class_fixture() - ob2 = FunctionsTest.create_instance(InterfaceTestClass) + InterfaceTestClass = interface_test_class_fixture(test_create_instance.__name__) + ob2 = FunctionsTest.create_instance_interface(InterfaceTestClass) assert ob2.foo() == "InterfaceTestClass" assert FunctionsTest.test_foo(ob2) == "InterfaceTestClass" assert ob2.bar("bar", 2) == "bar/bar" assert FunctionsTest.test_bar(ob2, "bar", 2) == "bar/bar" - y = FunctionsTest.pass_through(ob2) - assert id(y) == id(ob2) + y = FunctionsTest.pass_through_interface(ob2) + assert id(y) != id(ob2) -@pytest.mark.skip(reason="FIXME: test randomly pass/fails") def test_events(): class EventHandler(object): def handler(self, x, args): @@ -169,12 +197,12 @@ def handler(self, x, args): assert FunctionsTest.test_event(x, 1) == 1 assert event_handler.value == 1 - InterfaceTestClass = interface_test_class_fixture() + InterfaceTestClass = interface_test_class_fixture(test_events.__name__) i = InterfaceTestClass() with pytest.raises(System.NotImplementedException): FunctionsTest.test_event(i, 2) - DerivedEventTest = derived_event_test_class_fixture() + DerivedEventTest = derived_event_test_class_fixture(test_events.__name__) d = DerivedEventTest() d.add_TestEvent(event_handler.handler) assert FunctionsTest.test_event(d, 3) == 3 @@ -193,3 +221,59 @@ def test_isinstance_check(): for x in b: assert isinstance(x, System.Object) assert isinstance(x, System.String) + +def test_namespace_and_init(): + calls = [] + class TestX(System.Object): + __namespace__ = "test_clr_subclass_with_init_args" + def __init__(self, *args, **kwargs): + calls.append((args, kwargs)) + t = TestX(1,2,3,foo="bar") + assert len(calls) == 1 + assert calls[0][0] == (1,2,3) + assert calls[0][1] == {"foo":"bar"} + +def test_namespace_and_argless_init(): + calls = [] + class TestX(System.Object): + __namespace__ = "test_clr_subclass_without_init_args" + def __init__(self): + calls.append(True) + t = TestX() + assert len(calls) == 1 + assert calls[0] == True + + +def test_namespace_and_no_init(): + class TestX(System.Object): + __namespace__ = "test_clr_subclass_without_init" + q = 1 + t = TestX() + assert t.q == 1 + +def test_construction_from_clr(): + import clr + calls = [] + class TestX(System.Object): + __namespace__ = "test_clr_subclass_init_from_clr" + @clr.clrmethod(None, [int, str]) + def __init__(self, i, s): + calls.append((i, s)) + + # Construct a TestX from Python + t = TestX(1, "foo") + assert len(calls) == 1 + assert calls[0][0] == 1 + assert calls[0][1] == "foo" + + # Reset calls and construct a TestX from CLR + calls = [] + tp = t.GetType() + t2 = tp.GetConstructors()[0].Invoke(None) + assert len(calls) == 0 + + # The object has only been constructed, now it needs to be initialized as well + tp.GetMethod("__init__").Invoke(t2, [1, "foo"]) + assert len(calls) == 1 + assert calls[0][0] == 1 + assert calls[0][1] == "foo" diff --git a/src/tests/test_sysargv.py b/src/tests/test_sysargv.py index d86aa1c1d..dd62bc632 100644 --- a/src/tests/test_sysargv.py +++ b/src/tests/test_sysargv.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- - """Test sys.argv state.""" import sys - -from ._compat import check_output +from subprocess import check_output def test_sys_argv_state(filepath): @@ -14,5 +11,5 @@ def test_sys_argv_state(filepath): script = filepath("argv-fixture.py") out = check_output([sys.executable, script, "foo", "bar"]) - assert "foo" in out - assert "bar" in out + assert b"foo" in out + assert b"bar" in out diff --git a/src/tests/test_thread.py b/src/tests/test_thread.py index c62c15939..909c71f1c 100644 --- a/src/tests/test_thread.py +++ b/src/tests/test_thread.py @@ -5,7 +5,7 @@ import threading import time -from ._compat import range, thread +import _thread as thread from .utils import dprint diff --git a/src/tests/tests.pyproj b/src/tests/tests.pyproj index cefa0d05e..4bdbc6b14 100644 --- a/src/tests/tests.pyproj +++ b/src/tests/tests.pyproj @@ -28,7 +28,6 @@ - @@ -59,6 +58,7 @@ + diff --git a/src/tests/utils.py b/src/tests/utils.py index cacb015ec..b467cae97 100644 --- a/src/tests/utils.py +++ b/src/tests/utils.py @@ -5,9 +5,7 @@ Refactor utility functions and classes """ -from __future__ import print_function - -from ._compat import PY2, PY3 +DictProxyType = type(object.__dict__) def dprint(msg): @@ -21,11 +19,8 @@ def is_clr_module(ob): def is_clr_root_module(ob): - if PY3: - # in Python 3 the clr module is a normal python module - return ob.__name__ == "clr" - elif PY2: - return type(ob).__name__ == 'CLRModule' + # in Python 3 the clr module is a normal python module + return ob.__name__ == "clr" def is_clr_class(ob): diff --git a/tools/geninterop/fake_libc_include/crypt.h b/tools/geninterop/fake_libc_include/crypt.h new file mode 100644 index 000000000..3b16d481f --- /dev/null +++ b/tools/geninterop/fake_libc_include/crypt.h @@ -0,0 +1 @@ +#include "features.h" \ No newline at end of file diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index bf5fdb96b..0d5b83b30 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -21,6 +21,11 @@ import sysconfig import subprocess +if sys.version_info.major > 2: + from io import StringIO +else: + from StringIO import StringIO + from pycparser import c_ast, c_parser _log = logging.getLogger() @@ -48,22 +53,25 @@ class AstParser(object): """Walk an AST and determine the members of all structs""" def __init__(self): - self.__typedefs = {} - self.__typedecls = {} - self.__structs = {} - self.__struct_stack = [] - self.__struct_members_stack = [] - self.__ptr_decl_depth = 0 - self.__struct_members = {} + self._typedefs = {} + self._typedecls = {} + self._structs = {} + self._struct_stack = [] + self._struct_members_stack = [] + self._ptr_decl_depth = 0 + self._struct_members = {} + self._decl_names = {} def get_struct_members(self, name): """return a list of (name, type) of struct members""" - if name in self.__typedefs: - node = self.__get_leaf_node(self.__typedefs[name]) - name = node.name - if name not in self.__struct_members: - raise Exception("Unknown struct '%s'" % name) - return self.__struct_members[name] + defs = self._typedefs.get(name) + if defs is None: + return None + node = self._get_leaf_node(defs) + name = node.name + if name is None: + name = defs.declname + return self._struct_members.get(name) def visit(self, node): if isinstance(node, c_ast.FileAST): @@ -76,6 +84,8 @@ def visit(self, node): self.visit_struct(node) elif isinstance(node, c_ast.Decl): self.visit_decl(node) + elif isinstance(node, c_ast.FuncDecl): + self.visit_funcdecl(node) elif isinstance(node, c_ast.PtrDecl): self.visit_ptrdecl(node) elif isinstance(node, c_ast.IdentifierType): @@ -86,76 +96,101 @@ def visit_ast(self, ast): self.visit(node) def visit_typedef(self, typedef): - self.__typedefs[typedef.name] = typedef.type + self._typedefs[typedef.name] = typedef.type self.visit(typedef.type) def visit_typedecl(self, typedecl): + self._decl_names[typedecl.type] = typedecl.declname self.visit(typedecl.type) def visit_struct(self, struct): - self.__structs[self.__get_struct_name(struct)] = struct if struct.decls: + self._structs[self._get_struct_name(struct)] = struct # recurse into the struct - self.__struct_stack.insert(0, struct) + self._struct_stack.insert(0, struct) for decl in struct.decls: - self.__struct_members_stack.insert(0, decl.name) + self._struct_members_stack.insert(0, decl.name) self.visit(decl) - self.__struct_members_stack.pop(0) - self.__struct_stack.pop(0) - elif self.__ptr_decl_depth: + self._struct_members_stack.pop(0) + self._struct_stack.pop(0) + elif self._ptr_decl_depth: # the struct is empty, but add it as a member to the current # struct as the current member maybe a pointer to it. - self.__add_struct_member(struct.name) + self._add_struct_member(struct.name) def visit_decl(self, decl): self.visit(decl.type) + def visit_funcdecl(self, funcdecl): + self.visit(funcdecl.type) + def visit_ptrdecl(self, ptrdecl): - self.__ptr_decl_depth += 1 + self._ptr_decl_depth += 1 self.visit(ptrdecl.type) - self.__ptr_decl_depth -= 1 + self._ptr_decl_depth -= 1 def visit_identifier(self, identifier): type_name = " ".join(identifier.names) - self.__add_struct_member(type_name) + self._add_struct_member(type_name) - def __add_struct_member(self, type_name): - if not (self.__struct_stack and self.__struct_members_stack): + def _add_struct_member(self, type_name): + if not (self._struct_stack and self._struct_members_stack): return # add member to current struct - current_struct = self.__struct_stack[0] - member_name = self.__struct_members_stack[0] - struct_members = self.__struct_members.setdefault( - self.__get_struct_name(current_struct), []) + current_struct = self._struct_stack[0] + member_name = self._struct_members_stack[0] + struct_members = self._struct_members.setdefault( + self._get_struct_name(current_struct), []) # get the node associated with this type node = None - if type_name in self.__typedefs: - node = self.__get_leaf_node(self.__typedefs[type_name]) - elif type_name in self.__structs: - node = self.__structs[type_name] + if type_name in self._typedefs: + node = self._get_leaf_node(self._typedefs[type_name]) + # If the struct was only declared when the typedef was created, its member + # information will not have been recorded and we have to look it up in the + # structs + if isinstance(node, c_ast.Struct) and node.decls is None: + if node.name in self._structs: + node = self._structs[node.name] + elif type_name in self._structs: + node = self._structs[type_name] # If it's a struct (and not a pointer to a struct) expand # it into the current struct definition - if not self.__ptr_decl_depth and isinstance(node, c_ast.Struct): + if not self._ptr_decl_depth and isinstance(node, c_ast.Struct): for decl in node.decls or []: - self.__struct_members_stack.insert(0, decl.name) + self._struct_members_stack.insert(0, decl.name) self.visit(decl) - self.__struct_members_stack.pop(0) + self._struct_members_stack.pop(0) else: # otherwise add it as a single member struct_members.append((member_name, type_name)) - def __get_leaf_node(self, node): + def _get_leaf_node(self, node): if isinstance(node, c_ast.Typedef): - return self.__get_leaf_node(node.type) + return self._get_leaf_node(node.type) if isinstance(node, c_ast.TypeDecl): - return self.__get_leaf_node(node.type) + return self._get_leaf_node(node.type) return node - def __get_struct_name(self, node): - return node.name or "_struct_%d" % id(node) + def _get_struct_name(self, node): + return node.name or self._decl_names.get(node) or "_struct_%d" % id(node) + + +class Writer(object): + + def __init__(self): + self._stream = StringIO() + + def append(self, indent=0, code=""): + self._stream.write("%s%s\n" % (indent * " ", code)) + + def extend(self, s): + self._stream.write(s) + + def to_string(self): + return self._stream.getvalue() def preprocess_python_headers(): @@ -169,13 +204,24 @@ def preprocess_python_headers(): include_py = sysconfig.get_config_var("INCLUDEPY") include_dirs.append(include_py) + include_args = [c for p in include_dirs for c in ["-I", p]] + defines = [ "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", - "-D", "__int64=long long" + "-D", "__int64=long long", + "-D", "_POSIX_THREADS" ] + if os.name == 'nt': + defines.extend([ + "-D", "__inline=inline", + "-D", "__ptr32=", + "-D", "__ptr64=", + "-D", "__declspec(x)=", + ]) + if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) @@ -185,7 +231,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = [] @@ -196,77 +242,78 @@ def preprocess_python_headers(): return "\n".join(lines) -def gen_interop_code(members): - """Generate the TypeOffset C# class""" - - defines = [ - "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR) - ] - - if hasattr(sys, "abiflags"): - if "d" in sys.abiflags: - defines.append("PYTHON_WITH_PYDEBUG") - if "m" in sys.abiflags: - defines.append("PYTHON_WITH_PYMALLOC") - if "u" in sys.abiflags: - defines.append("PYTHON_WITH_WIDE_UNICODE") +def gen_interop_head(writer): filename = os.path.basename(__file__) - defines_str = " && ".join(defines) + abi_flags = getattr(sys, "abiflags", "") + py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR) class_definition = """ // Auto-generated by %s. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. +// Python %s: ABI flags: '%s' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo -#if %s using System; -using System.Collections; -using System.Collections.Specialized; +using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; + +using Python.Runtime.Native; namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } +{""" % (filename, py_ver, abi_flags) + writer.extend(class_definition) + + +def gen_interop_tail(writer): + tail = """} +""" + writer.extend(tail) + +def gen_heap_type_members(parser, writer, type_name = None): + """Generate the TypeOffset C# class""" + members = parser.get_struct_members("PyHeapTypeObject") + type_name = type_name or "TypeOffset{0}{1}".format(PY_MAJOR, PY_MINOR) + class_definition = """ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class {0} : GeneratedTypeOffsets, ITypeOffsets + {{ + public {0}() {{ }} // Auto-generated from PyHeapTypeObject in Python.h -""" % (filename, defines_str) +""".format(type_name) # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. for name, tpy in members: name = _typeoffset_member_renames.get(name, name) - class_definition += " public static int %s = 0;\n" % name - - class_definition += """ - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} + class_definition += " public int %s { get; private set; }\n" % name -#endif + class_definition += """ } """ - return class_definition + writer.extend(class_definition) +def gen_structure_code(parser, writer, type_name, indent): + members = parser.get_struct_members(type_name) + if members is None: + return False + out = writer.append + out(indent, "[StructLayout(LayoutKind.Sequential)]") + out(indent, "internal struct %s" % type_name) + out(indent, "{") + for name, tpy in members: + out(indent + 1, "public IntPtr %s;" % name) + out(indent, "}") + out() + return True + def main(): # preprocess Python.h and build the AST python_h = preprocess_python_headers() @@ -277,10 +324,16 @@ def main(): ast_parser = AstParser() ast_parser.visit(ast) + writer = Writer() # generate the C# code - members = ast_parser.get_struct_members("PyHeapTypeObject") - interop_cs = gen_interop_code(members) + offsets_type_name = "NativeTypeOffset" if len(sys.argv) > 1 else None + gen_interop_head(writer) + + gen_heap_type_members(ast_parser, writer, type_name = offsets_type_name) + + gen_interop_tail(writer) + interop_cs = writer.to_string() if len(sys.argv) > 1: with open(sys.argv[1], "w") as fh: fh.write(interop_cs) diff --git a/tools/nuget/nuget.exe b/tools/nuget/nuget.exe deleted file mode 100644 index e42e6d827..000000000 Binary files a/tools/nuget/nuget.exe and /dev/null differ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 3c9be5c63..000000000 --- a/tox.ini +++ /dev/null @@ -1,27 +0,0 @@ -[tox] -skip_missing_interpreters=True -envlist = - py27 - py33 - py34 - py35 - py36 - py37 - check - -[testenv] -deps = - pytest -commands = - {posargs:py.test} - -[testenv:check] -ignore_errors=True -skip_install=True -basepython=python3.5 -deps = - check-manifest - flake8 -commands = - flake8 src setup.py - python setup.py check --strict --metadata