Skip to content

Commit 2bf6a4b

Browse files
ritwik-gpre-commit-ci[bot]
ritwik-g
and
pre-commit-ci[bot]
authored
OTEL instrumentation Support (Zipstack#1218)
* Added OTEL instrumentation dependencies for tracing * Moved gunicorn to main dependency and removed the version for otel instrumentations * Commit pdm.lock changes * Changed the deploy group from dev to optional * Commit pdm.lock changes * Corrected optional dependencies * Commit pdm.lock changes * Added otel config in all services. Added additional env logic in runner to support otel config passing * Commit pdm.lock changes * Corrected logs and runner * corrected the wrong method name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * reverted adding the otel ids since it causes errro when isntrumentation not present * fixes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Added missing dependency for tracing * Commit pdm.lock changes * Trace context support for tool. Added logs with trace id in flask logging helper * Fix line length * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed line length * Commit pdm.lock changes * Added version for otel sdk and apis * Apply suggestions from code review Signed-off-by: Ritwik G <100672805+ritwik-g@users.noreply.github.com> * Update tools/structure/src/main.py Signed-off-by: Ritwik G <100672805+ritwik-g@users.noreply.github.com> * Removed instrumentation dependencies * Commit pdm.lock changes * Commit pdm.lock changes --------- Signed-off-by: Ritwik G <100672805+ritwik-g@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 87049bd commit 2bf6a4b

19 files changed

Lines changed: 3269 additions & 1580 deletions

File tree

‎backend/backend/settings/base.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,14 @@ def get_required_setting(
156156
CSRF_TRUSTED_ORIGINS = [WEB_APP_ORIGIN_URL]
157157
CORS_ALLOW_ALL_ORIGINS = False
158158

159+
# Determine if OpenTelemetry trace context should be included in logs
160+
OTEL_TRACE_CONTEXT = (
161+
" trace_id:%(otelTraceID)s span_id:%(otelSpanID)s"
162+
if os.environ.get("OTEL_TRACES_EXPORTER", "none").lower() != "none"
163+
else ""
164+
)
165+
166+
# Logging configuration
159167
LOGGING = {
160168
"version": 1,
161169
"disable_existing_loggers": False,
@@ -167,7 +175,9 @@ def get_required_setting(
167175
"format": (
168176
"%(levelname)s : [%(asctime)s]"
169177
"{module:%(module)s process:%(process)d "
170-
"thread:%(thread)d request_id:%(request_id)s} :- %(message)s"
178+
"thread:%(thread)d request_id:%(request_id)s"
179+
+ OTEL_TRACE_CONTEXT
180+
+ "} :- %(message)s"
171181
),
172182
},
173183
"verbose": {

‎backend/pdm.lock‎

Lines changed: 732 additions & 427 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎backend/pyproject.toml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,6 @@ classifiers = [
6868
]
6969

7070
[tool.pdm.dev-dependencies]
71-
deploy = [
72-
"gunicorn>=21.2.0",
73-
]
7471
test = [
7572
"pytest>=8.0.1",
7673
"pytest-dotenv==0.5.2",
@@ -87,6 +84,15 @@ dev = [
8784
"inotify>=0.2.10",
8885
]
8986

87+
[project.optional-dependencies]
88+
deploy = [
89+
"gunicorn~=23.0", # For serving the application
90+
# Keep versions empty and let pdm decide version
91+
# since we use no code instrumentation and don't use in code
92+
"opentelemetry-distro",
93+
"opentelemetry-exporter-otlp",
94+
]
95+
9096
[tool.pytest.ini_options]
9197
env_files = "test.env" # Load env from particular env file
9298
addopts = "-s"

‎docker/dockerfiles/backend.Dockerfile‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ ENV \
1212
BUILD_PACKAGES_PATH=unstract \
1313
DJANGO_SETTINGS_MODULE="backend.settings.dev" \
1414
PDM_VERSION=2.16.1 \
15-
# Disable all telemetry by default
16-
OTEL_TRACES_EXPORTER=none \
15+
# OpenTelemetry configuration (disabled by default, enable in docker-compose)
1716
OTEL_TRACES_EXPORTER=none \
17+
OTEL_METRICS_EXPORTER=none \
1818
OTEL_LOGS_EXPORTER=none \
1919
OTEL_SERVICE_NAME=unstract_backend
2020

@@ -33,21 +33,16 @@ WORKDIR /app
3333

3434
# Create venv and install gunicorn and other deps in it
3535
RUN pdm venv create -w virtualenv --with-pip && \
36-
. .venv/bin/activate && \
37-
pip install --no-cache-dir \
38-
gunicorn \
39-
# Install opentelemetry for instrumentation
40-
opentelemetry-distro \
41-
opentelemetry-exporter-otlp && \
42-
opentelemetry-bootstrap -a install
36+
. .venv/bin/activate
4337

4438
COPY ${BUILD_CONTEXT_PATH}/ /app/
4539
# Copy local dependency packages
4640
COPY ${BUILD_PACKAGES_PATH}/ /unstract
4741

4842
# Install dependencies
4943
RUN . .venv/bin/activate && \
50-
pdm sync --prod --no-editable
44+
pdm sync --prod --no-editable --with deploy && \
45+
opentelemetry-bootstrap -a install
5146

5247
EXPOSE 8000
5348

‎docker/dockerfiles/platform.Dockerfile‎

Lines changed: 56 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,56 @@
1-
FROM python:3.9-slim
2-
3-
LABEL maintainer="Zipstack Inc."
4-
5-
ENV \
6-
# Keeps Python from generating .pyc files in the container
7-
PYTHONDONTWRITEBYTECODE=1 \
8-
# Set to immediately flush stdout and stderr streams without first buffering
9-
PYTHONUNBUFFERED=1 \
10-
PYTHONPATH=/unstract \
11-
BUILD_CONTEXT_PATH=platform-service \
12-
BUILD_PACKAGES_PATH=unstract \
13-
PDM_VERSION=2.16.1
14-
15-
# Install system dependencies
16-
RUN apt-get update; \
17-
apt-get --no-install-recommends install -y \
18-
# unstract sdk
19-
build-essential libmagic-dev; \
20-
\
21-
apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; \
22-
\
23-
pip install --no-cache-dir -U pip pdm~=${PDM_VERSION}; \
24-
\
25-
# Creates a non-root user with an explicit UID and adds permission to access the /app folder
26-
# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
27-
adduser -u 5678 --disabled-password --gecos "" unstract;
28-
29-
USER unstract
30-
31-
WORKDIR /app
32-
33-
# Create venv and install gunicorn and other deps in it
34-
RUN pdm venv create -w virtualenv --with-pip && \
35-
. .venv/bin/activate && \
36-
pip install --no-cache-dir \
37-
gunicorn \
38-
# Install opentelemetry for instrumentation
39-
opentelemetry-distro \
40-
opentelemetry-exporter-otlp && \
41-
opentelemetry-bootstrap -a install
42-
43-
# Read and execute access to non-root user to avoid security hotspot
44-
# Write access to specific sub-directory need to be explicitly provided if required
45-
COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/
46-
# Copy local dependency packages
47-
COPY --chown=unstract ${BUILD_PACKAGES_PATH} /unstract
48-
49-
# Install dependencies
50-
RUN . .venv/bin/activate && \
51-
pdm sync --prod --no-editable
52-
53-
EXPOSE 3001
54-
55-
# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
56-
CMD [".venv/bin/gunicorn", "--bind", "0.0.0.0:3001", "--timeout", "300", "unstract.platform_service.run:app"]
1+
FROM python:3.9-slim
2+
3+
LABEL maintainer="Zipstack Inc."
4+
5+
ENV \
6+
# Keeps Python from generating .pyc files in the container
7+
PYTHONDONTWRITEBYTECODE=1 \
8+
# Set to immediately flush stdout and stderr streams without first buffering
9+
PYTHONUNBUFFERED=1 \
10+
PYTHONPATH=/unstract \
11+
BUILD_CONTEXT_PATH=platform-service \
12+
BUILD_PACKAGES_PATH=unstract \
13+
PDM_VERSION=2.16.1 \
14+
# OpenTelemetry configuration (disabled by default, enable in docker-compose)
15+
OTEL_TRACES_EXPORTER=none \
16+
OTEL_METRICS_EXPORTER=none \
17+
OTEL_LOGS_EXPORTER=none \
18+
OTEL_SERVICE_NAME=unstract_platform
19+
20+
# Install system dependencies
21+
RUN apt-get update; \
22+
apt-get --no-install-recommends install -y \
23+
# unstract sdk
24+
build-essential libmagic-dev; \
25+
\
26+
apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; \
27+
\
28+
pip install --no-cache-dir -U pip pdm~=${PDM_VERSION}; \
29+
\
30+
# Creates a non-root user with an explicit UID and adds permission to access the /app folder
31+
# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
32+
adduser -u 5678 --disabled-password --gecos "" unstract;
33+
34+
USER unstract
35+
36+
WORKDIR /app
37+
38+
# Create venv and install gunicorn and other deps in it
39+
RUN pdm venv create -w virtualenv --with-pip && \
40+
. .venv/bin/activate
41+
42+
# Read and execute access to non-root user to avoid security hotspot
43+
# Write access to specific sub-directory need to be explicitly provided if required
44+
COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/
45+
# Copy local dependency packages
46+
COPY --chown=unstract ${BUILD_PACKAGES_PATH} /unstract
47+
48+
# Install dependencies
49+
RUN . .venv/bin/activate && \
50+
pdm sync --prod --no-editable --with deploy && \
51+
opentelemetry-bootstrap -a install
52+
53+
EXPOSE 3001
54+
55+
# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
56+
CMD [".venv/bin/gunicorn", "--bind", "0.0.0.0:3001", "--timeout", "300", "unstract.platform_service.run:app"]

‎docker/dockerfiles/prompt.Dockerfile‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,12 @@ ENV \
1111
BUILD_CONTEXT_PATH=prompt-service \
1212
BUILD_PACKAGES_PATH=unstract \
1313
TARGET_PLUGINS_PATH=src/unstract/prompt_service/plugins \
14-
PDM_VERSION=2.16.1
14+
PDM_VERSION=2.16.1 \
15+
# OpenTelemetry configuration (disabled by default, enable in docker-compose)
16+
OTEL_TRACES_EXPORTER=none \
17+
OTEL_METRICS_EXPORTER=none \
18+
OTEL_LOGS_EXPORTER=none \
19+
OTEL_SERVICE_NAME=unstract_prompt
1520

1621
# Install system dependencies
1722
RUN apt-get update; \
@@ -34,14 +39,7 @@ WORKDIR /app
3439

3540
# Create venv and install gunicorn and other deps in it
3641
RUN pdm venv create -w virtualenv --with-pip && \
37-
. .venv/bin/activate && \
38-
pip install --no-cache-dir \
39-
gunicorn \
40-
gevent \
41-
# Install opentelemetry for instrumentation
42-
opentelemetry-distro \
43-
opentelemetry-exporter-otlp && \
44-
opentelemetry-bootstrap -a install
42+
. .venv/bin/activate
4543

4644
# TODO: Security issue but ignoring it for nuitka based builds
4745
COPY --chown=unstract ${BUILD_CONTEXT_PATH} /app/
@@ -51,7 +49,7 @@ COPY --chown=unstract ${BUILD_PACKAGES_PATH}/flags /unstract/flags
5149

5250
# Install dependencies and plugins (if any)
5351
RUN . .venv/bin/activate && \
54-
pdm sync --prod --no-editable && \
52+
pdm sync --prod --no-editable --with deploy && \
5553
for dir in "${TARGET_PLUGINS_PATH}"/*/; do \
5654
dirpath=${dir%*/}; \
5755
if [ "${dirpath##*/}" != "*" ]; then \
@@ -61,7 +59,8 @@ RUN . .venv/bin/activate && \
6159
cd -; \
6260
fi; \
6361
done && \
64-
mkdir prompt-studio-data
62+
mkdir prompt-studio-data && \
63+
opentelemetry-bootstrap -a install
6564

6665
EXPOSE 3003
6766

‎docker/dockerfiles/runner.Dockerfile‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ ENV \
1010
PYTHONPATH=/unstract \
1111
BUILD_CONTEXT_PATH=runner \
1212
BUILD_PACKAGES_PATH=unstract \
13-
PDM_VERSION=2.16.1
13+
PDM_VERSION=2.16.1 \
14+
# OpenTelemetry configuration (disabled by default, enable in docker-compose)
15+
OTEL_TRACES_EXPORTER=none \
16+
OTEL_METRICS_EXPORTER=none \
17+
OTEL_LOGS_EXPORTER=none \
18+
OTEL_SERVICE_NAME=unstract_runner
1419

1520
RUN apt-get update \
1621
&& apt-get --no-install-recommends install -y docker \
@@ -22,14 +27,7 @@ WORKDIR /app
2227

2328
# Create venv and install gunicorn and other deps in it
2429
RUN pdm venv create -w virtualenv --with-pip && \
25-
. .venv/bin/activate && \
26-
pip install --no-cache-dir \
27-
gunicorn \
28-
gevent \
29-
# Install opentelemetry for instrumentation
30-
opentelemetry-distro \
31-
opentelemetry-exporter-otlp && \
32-
opentelemetry-bootstrap -a install
30+
. .venv/bin/activate
3331

3432
COPY ${BUILD_CONTEXT_PATH} /app/
3533
# Copy local dependency packages
@@ -40,13 +38,15 @@ RUN \
4038
# source command may not be availble in sh
4139
. .venv/bin/activate; \
4240
\
43-
pdm sync --prod --no-editable; \
41+
pdm sync --prod --no-editable --with deploy; \
4442
\
4543
if [ -f cloud_requirements.txt ]; then \
4644
pip install --no-cache-dir -r cloud_requirements.txt; \
4745
else \
4846
echo "cloud_requirements.txt does not exist"; \
49-
fi
47+
fi; \
48+
\
49+
opentelemetry-bootstrap -a install
5050

5151
EXPOSE 5002
5252

‎docker/dockerfiles/x2text.Dockerfile‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@ ENV \
88
# Set to immediately flush stdout and stderr streams without first buffering
99
PYTHONUNBUFFERED=1 \
1010
BUILD_CONTEXT_PATH=x2text-service \
11-
PDM_VERSION=2.16.1
11+
PDM_VERSION=2.16.1 \
12+
# OpenTelemetry configuration (disabled by default, enable in docker-compose)
13+
OTEL_TRACES_EXPORTER=none \
14+
OTEL_METRICS_EXPORTER=none \
15+
OTEL_LOGS_EXPORTER=none \
16+
OTEL_SERVICE_NAME=unstract_x2text
1217

1318
RUN pip install --no-cache-dir -U pip pdm~=${PDM_VERSION}; \
1419
\
@@ -22,21 +27,16 @@ WORKDIR /app
2227

2328
# Create venv and install gunicorn and other deps in it
2429
RUN pdm venv create -w virtualenv --with-pip && \
25-
. .venv/bin/activate && \
26-
pip install --no-cache-dir \
27-
gunicorn \
28-
# Install opentelemetry for instrumentation
29-
opentelemetry-distro \
30-
opentelemetry-exporter-otlp && \
31-
opentelemetry-bootstrap -a install
30+
. .venv/bin/activate
3231

3332
# Read and execute access to non-root user to avoid security hotspot
3433
# Write access to specific sub-directory need to be explicitly provided if required
3534
COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/
3635

3736
# Install dependencies
3837
RUN . .venv/bin/activate && \
39-
pdm sync --prod --no-editable
38+
pdm sync --prod --no-editable --with deploy && \
39+
opentelemetry-bootstrap -a install
4040

4141
EXPOSE 3004
4242

0 commit comments

Comments
 (0)