diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 0000000..a139bff --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,3 @@ +{ + extends: ["github>cloudquery/.github//.github/renovate-python-default.json5"], +} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7ce2828 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,21 @@ +name: Lint with Black + +on: + pull_request: + push: + branches: + - main + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Check formatting + run: make fmt-check diff --git a/.github/workflows/pr_title.yml b/.github/workflows/pr_title.yml new file mode 100644 index 0000000..c44b736 --- /dev/null +++ b/.github/workflows/pr_title.yml @@ -0,0 +1,52 @@ +name: "Validate PR title" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + # Please look up the latest version from + # https://github.com/amannn/action-semantic-pull-request/releases + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure which types are allowed. + # Default: https://github.com/commitizen/conventional-commit-types + types: | + fix + feat + chore + refactor + test + # Configure that a scope must always be provided. + requireScope: false + # Configure additional validation for the subject based on a regex. + # This example ensures the subject starts with an uppercase character. + subjectPattern: ^[A-Z].+$ + # If `subjectPattern` is configured, you can use this property to override + # the default error message that is shown when the pattern doesn't match. + # The variables `subject` and `title` can be used within the message. + subjectPatternError: | + The subject "{subject}" found in the pull request title "{title}" + didn't match the configured pattern. Please ensure that the subject + starts with an uppercase character. + # For work-in-progress PRs you can typically use draft pull requests + # from Github. However, private repositories on the free plan don't have + # this option and therefore this action allows you to opt-in to using the + # special "[WIP]" prefix to indicate this state. This will avoid the + # validation of the PR title and the pull request checks remain pending. + # Note that a second check will be reported if this is enabled. + wip: true + # When using "Squash and merge" on a PR with only one commit, GitHub + # will suggest using that commit message instead of the PR title for the + # merge commit, and it's easy to commit this by mistake. Enable this option + # to also validate the commit message for one commit PRs. + validateSingleCommit: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c0dd026..858503f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,6 @@ jobs: python-version: '3.x' - name: Install dependencies run: | - python -m pip install --upgrade pip pip install -r requirements.txt pip install build - name: Build package diff --git a/.github/workflows/regen.yml b/.github/workflows/regen.yml new file mode 100644 index 0000000..ab67c80 --- /dev/null +++ b/.github/workflows/regen.yml @@ -0,0 +1,36 @@ +name: Generate Python Code from plugin-pb +on: + schedule: + - cron: "0 8 * * *" + workflow_dispatch: + +jobs: + regen: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: | + pip install -r requirements.txt + - name: Generate code + run: | + make clone-proto + make gen-proto + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + # required so the PR triggers workflow runs + token: ${{ secrets.GH_CQ_BOT }} + branch: fix/gen_proto + base: main + title: "fix: Generate Python Code from `plugin-pb`" + commit-message: "fix: Generate Python Code from `plugin-pb`" + body: This PR was created by a scheduled workflow to regenerate the Python code from `plugin-pb`. + author: cq-bot + labels: automerge diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 0000000..fd91639 --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,23 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: pip install -r requirements.txt + - name: Run tests + run: make test diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..78a6a2b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## [0.0.15](https://github.com/cloudquery/plugin-pb-python/compare/v0.0.14...v0.0.15) (2023-08-08) + + +### Bug Fixes + +* **deps:** Update dependency grpcio to v1.56.2 ([#8](https://github.com/cloudquery/plugin-pb-python/issues/8)) ([5330bc4](https://github.com/cloudquery/plugin-pb-python/commit/5330bc450ba82a453ef0d44936688650ed9748dc)) +* **deps:** Update dependency grpcio-tools to v1.56.2 ([#9](https://github.com/cloudquery/plugin-pb-python/issues/9)) ([66b4add](https://github.com/cloudquery/plugin-pb-python/commit/66b4addefa2e337b5a7960d902460bcc53df7254)) +* Generate Python Code from `plugin-pb` ([#13](https://github.com/cloudquery/plugin-pb-python/issues/13)) ([29c0241](https://github.com/cloudquery/plugin-pb-python/commit/29c0241d7229eb45f558dff4f7b791ffe77b9f76)) +* Generate Python Code from `plugin-pb` ([#14](https://github.com/cloudquery/plugin-pb-python/issues/14)) ([b5be5c6](https://github.com/cloudquery/plugin-pb-python/commit/b5be5c64442791a5fa5b71176112fd0449ae5c34)) diff --git a/Makefile b/Makefile index 79c1a70..a4f0c40 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,16 @@ +test: + pytest . + +fmt: + black . --exclude=cloudquery + +fmt-check: + black --check . --exclude=cloudquery clone-proto: git clone https://github.com/cloudquery/plugin-pb -gen: +gen-proto: cd plugin-pb && git pull && cd .. mkdir -p ./protos/cloudquery/plugin_v3 diff --git a/README.md b/README.md index 10f66f5..0239893 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # plugin-pb-python -This is a low-level auto-generate gRPC client and server for CloudQuery plugin from [plugin-pb protos](https://github.com/cloudquery/plugin-pb). +This is a low-level auto-generated gRPC client and server for CloudQuery plugin from [plugin-pb protos](https://github.com/cloudquery/plugin-pb). ## Development diff --git a/cloudquery/discovery_v1/discovery_pb2.py b/cloudquery/discovery_v1/discovery_pb2.py index ad78cf8..0f4806a 100644 --- a/cloudquery/discovery_v1/discovery_pb2.py +++ b/cloudquery/discovery_v1/discovery_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\'cloudquery/discovery_v1/discovery.proto\x12\x17\x63loudquery.discovery.v1\"6\n\x0bGetVersions\x1a\t\n\x07Request\x1a\x1c\n\x08Response\x12\x10\n\x08versions\x18\x01 \x03(\x05\x32w\n\tDiscovery\x12j\n\x0bGetVersions\x12,.cloudquery.discovery.v1.GetVersions.Request\x1a-.cloudquery.discovery.v1.GetVersions.ResponseB>Z None: ... + skip_dependent_tables: bool + def __init__(self, tables: _Optional[_Iterable[str]] = ..., skip_tables: _Optional[_Iterable[str]] = ..., skip_dependent_tables: bool = ...) -> None: ... class Response(_message.Message): __slots__ = ["tables"] TABLES_FIELD_NUMBER: _ClassVar[int] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..7dab082 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = tests/*.py diff --git a/requirements.txt b/requirements.txt index 6314f5c..3beab03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -grpcio==1.56.0 -grpcio-tools==1.56.0 +black==23.7.0 +grpcio-tools==1.56.2 +grpcio==1.56.2 protobuf==4.23.4 +pyarrow==12.0.1 +pytest==7.4.0 \ No newline at end of file diff --git a/samples/plugin_client.py b/samples/plugin_client.py index c85e440..820db2a 100644 --- a/samples/plugin_client.py +++ b/samples/plugin_client.py @@ -2,15 +2,17 @@ import grpc from cloudquery.plugin_v3 import plugin_pb2, plugin_pb2_grpc + def run(): # NOTE(gRPC Python Team): .close() is possible on a channel and should be # used in circumstances in which the with statement does not fit the needs # of the code. - with grpc.insecure_channel('localhost:50051') as channel: + with grpc.insecure_channel("localhost:50051") as channel: stub = plugin_pb2_grpc.PluginStub(channel) response = stub.GetName(plugin_pb2.GetName.Request()) print(response.name) -if __name__ == '__main__': + +if __name__ == "__main__": logging.basicConfig() run() diff --git a/samples/plugin_server.py b/samples/plugin_server.py index db03900..8b82a6b 100644 --- a/samples/plugin_server.py +++ b/samples/plugin_server.py @@ -39,14 +39,13 @@ def Close(self, request, context): def serve(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - plugin_pb2_grpc.add_PluginServicer_to_server( - PluginServicer(), server) - server.add_insecure_port('[::]:50051') + plugin_pb2_grpc.add_PluginServicer_to_server(PluginServicer(), server) + server.add_insecure_port("[::]:50051") print("Starting server. Listening on port 50051") server.start() server.wait_for_termination() -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig() serve() diff --git a/setup.py b/setup.py index 00096cf..f3bc24d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ "grpcio >= 1.56.0", "grpcio-tools >= 1.56.0", "protobuf >= 4.23.4", - "pyarrow >= 12.0.1" + "pyarrow >= 12.0.1", ] url = "https://github.com/cloudquery/plugin-pb-python" @@ -37,7 +37,7 @@ ] setuptools.setup( name=name, - version="0.0.14", + version="0.0.15", description=description, long_description=long_description, author="CloudQuery LTD", @@ -65,6 +65,13 @@ # namespace_packages=namespaces, install_requires=dependencies, include_package_data=True, - package_data={"cloudquery": ["plugin_v3/py.typed", "plugin_v3/*.pyi", "discovery_v1/py.typed", "discovery_v1/*.pyi"]}, + package_data={ + "cloudquery": [ + "plugin_v3/py.typed", + "plugin_v3/*.pyi", + "discovery_v1/py.typed", + "discovery_v1/*.pyi", + ] + }, zip_safe=False, ) diff --git a/tests/plugin_v3/arrow.py b/tests/plugin_v3/arrow.py deleted file mode 100644 index 513207a..0000000 --- a/tests/plugin_v3/arrow.py +++ /dev/null @@ -1,16 +0,0 @@ -import pyarrow as pa -from cloudquery.plugin_v3.arrow import schemas_to_bytes, new_schemas_from_bytes, record_to_bytes, new_record_from_bytes - -def test_schema_round_trip(): - sc = pa.schema(fields=[pa.field("a", pa.int64())], metadata={"foo":"bar", "baz":"quux"}) - b = schemas_to_bytes([sc]) - schemas = new_schemas_from_bytes(b) - assert len(schemas) == 1 - assert schemas[0].equals(sc) - -def test_record_round_trip(): - sc = pa.schema(fields=[pa.field("a", pa.int64())], metadata={"foo":"bar", "baz":"quux"}) - rec = pa.RecordBatch.from_arrays([pa.array([1,2,3])], schema=sc) - b = record_to_bytes(rec) - rec2 = new_record_from_bytes(b) - assert rec.equals(rec2) diff --git a/tests/plugin_v3/arrow_test.py b/tests/plugin_v3/arrow_test.py new file mode 100644 index 0000000..c137c77 --- /dev/null +++ b/tests/plugin_v3/arrow_test.py @@ -0,0 +1,27 @@ +import pyarrow as pa +from cloudquery.plugin_v3.arrow import ( + schemas_to_bytes, + new_schemas_from_bytes, + record_to_bytes, + new_record_from_bytes, +) + + +def test_schema_round_trip(): + sc = pa.schema( + fields=[pa.field("a", pa.int64())], metadata={"foo": "bar", "baz": "quux"} + ) + b = schemas_to_bytes([sc]) + schemas = new_schemas_from_bytes(b) + assert len(schemas) == 1 + assert schemas[0].equals(sc) + + +def test_record_round_trip(): + sc = pa.schema( + fields=[pa.field("a", pa.int64())], metadata={"foo": "bar", "baz": "quux"} + ) + rec = pa.RecordBatch.from_arrays([pa.array([1, 2, 3])], schema=sc) + b = record_to_bytes(rec) + rec2 = new_record_from_bytes(b) + assert rec.equals(rec2)