# Copyright 2022 Google LLC

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

from __future__ import annotations

import os

import re

import subprocess

import sys

import textwrap

import uuid

from collections.abc import Callable, Iterable

from datetime import datetime

from typing import AsyncIterator

import pytest

import pytest_asyncio

def get_env_var(key: str) -> str:

v = os.environ.get(key)

if v is None:

raise ValueError(f"Must set env var {key}")

return v

@pytest.fixture(scope="session")

def table_name() -> str:

return "investments"

@pytest.fixture(scope="session")

def cluster_name() -> str:

return get_env_var("ALLOYDB_CLUSTER")

@pytest.fixture(scope="session")

def instance_name() -> str:

return get_env_var("ALLOYDB_INSTANCE")

@pytest.fixture(scope="session")

def region() -> str:

return get_env_var("ALLOYDB_REGION")

@pytest.fixture(scope="session")

def database_name() -> str:

return get_env_var("ALLOYDB_DATABASE_NAME")

@pytest.fixture(scope="session")

def password() -> str:

return get_env_var("ALLOYDB_PASSWORD")

@pytest_asyncio.fixture(scope="session")

def project_id() -> str:

gcp_project = get_env_var("GOOGLE_CLOUD_PROJECT")

run_cmd("gcloud", "config", "set", "project", gcp_project)

# Since everything requires the project, let's confiugre and show some

# debugging information here.

run_cmd("gcloud", "version")

run_cmd("gcloud", "config", "list")

return gcp_project

def run_cmd(*cmd: str) -> subprocess.CompletedProcess:

try:

print(f">> {cmd}")

start = datetime.now()

p = subprocess.run(

cmd,

check=True,

stdout=subprocess.PIPE,

stderr=subprocess.PIPE,

)

print(p.stderr.decode("utf-8"))

print(p.stdout.decode("utf-8"))

elapsed = (datetime.now() - start).seconds

minutes = int(elapsed / 60)

seconds = elapsed - minutes * 60

print(f"Command `{cmd[0]}` finished in {minutes}m {seconds}s")

return p

except subprocess.CalledProcessError as e:

# Include the error message from the failed command.

print(e.stderr.decode("utf-8"))

print(e.stdout.decode("utf-8"))

raise RuntimeError(f"{e}\n\n{e.stderr.decode('utf-8')}") from e

def run_notebook(

ipynb_file: str,

prelude: str = "",

section: str = "",

variables: dict = {},

replace: dict[str, str] = {},

preprocess: Callable[[str], str] = lambda source: source,

skip_shell_commands: bool = False,

until_end: bool = False,

) -> None:

import nbformat

from nbclient.client import NotebookClient

from nbclient.exceptions import CellExecutionError

def notebook_filter_section(

start: str,

end: str,

cells: list[nbformat.NotebookNode],

until_end: bool = False,

) -> Iterable[nbformat.NotebookNode]:

in_section = False

for cell in cells:

if cell["cell_type"] == "markdown":

if not in_section and cell["source"].startswith(start):

in_section = True

elif in_section and not until_end and cell["source"].startswith(end):

return

if in_section:

yield cell

# Regular expression to match and remove shell commands from the notebook.

# https://regex101.com/r/EHWBpT/1

shell_command_re = re.compile(r"^!((?:[^\n]+\\\n)*(?:[^\n]+))$", re.MULTILINE)

# Compile regular expressions for variable substitutions.

# https://regex101.com/r/e32vfW/1

compiled_substitutions = [

(

re.compile(rf"""\b{name}\s*=\s*(?:f?'[^']*'|f?"[^"]*"|\w+)"""),

f"{name} = {repr(value)}",

)

for name, value in variables.items()

]

# Filter the section if any, otherwise use the entire notebook.

nb = nbformat.read(ipynb_file, as_version=4)

if section:

start = section

end = section.split(" ", 1)[0] + " "

nb.cells = list(notebook_filter_section(start, end, nb.cells, until_end))

if len(nb.cells) == 0:

raise ValueError(

f"Section {repr(section)} not found in notebook {repr(ipynb_file)}"

)

# Preprocess the cells.

for cell in nb.cells:

# Only preprocess code cells.

if cell["cell_type"] != "code":

continue

# Run any custom preprocessing functions before.

cell["source"] = preprocess(cell["source"])

# Preprocess shell commands.

if skip_shell_commands:

cmd = "pass"

cell["source"] = shell_command_re.sub(cmd, cell["source"])

else:

cell["source"] = shell_command_re.sub(r"_run(f'''\1''')", cell["source"])

# Apply variable substitutions.

for regex, new_value in compiled_substitutions:

cell["source"] = regex.sub(new_value, cell["source"])

# Apply replacements.

for old, new in replace.items():

cell["source"] = cell["source"].replace(old, new)

# Clear outputs.

cell["outputs"] = []

# Prepend the prelude cell.

prelude_src = textwrap.dedent(

"""\

def _run(cmd):

import subprocess as _sp

import sys as _sys

_p = _sp.run(cmd, shell=True, stdout=_sp.PIPE, stderr=_sp.PIPE)

_stdout = _p.stdout.decode('utf-8').strip()

_stderr = _p.stderr.decode('utf-8').strip()

if _stdout:

print(f'➜ !{cmd}')

print(_stdout)

if _stderr:

print(f'➜ !{cmd}', file=_sys.stderr)

print(_stderr, file=_sys.stderr)

if _p.returncode:

raise RuntimeError('\\n'.join([

f"Command returned non-zero exit status {_p.returncode}.",

f"-------- command --------",

f"{cmd}",

f"-------- stderr --------",

f"{_stderr}",

f"-------- stdout --------",

f"{_stdout}",

]))

"""

+ prelude

)

nb.cells = [nbformat.v4.new_code_cell(prelude_src)] + nb.cells

# Run the notebook.

error = ""

client = NotebookClient(nb)

try:

client.execute()

except CellExecutionError as e:

# Remove colors and other escape characters to make it easier to read in the logs.

# https://stackoverflow.com/a/33925425

color_chars = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]")

error = color_chars.sub("", str(e))

for cell in nb.cells:

if cell["cell_type"] != "code":

continue

for output in cell["outputs"]:

if output.get("name") == "stdout":

print(color_chars.sub("", output["text"]))

elif output.get("name") == "stderr":

print(color_chars.sub("", output["text"]), file=sys.stderr)

if error:

raise RuntimeError(

f"Error on {repr(ipynb_file)}, section {repr(section)}: {error}"

)