From 2da7cd7e1728c964ad9fe8da72ebdf8564cb5844 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 27 Mar 2023 20:40:17 -0500 Subject: [PATCH 01/60] add pex_binary for st2 venv installation --- BUILD | 32 ++++++++++++++++++++++++++++++++ pants-plugins/macros.py | 10 ++++++++++ pants.toml | 2 ++ 3 files changed, 44 insertions(+) diff --git a/BUILD b/BUILD index f33988a645..610df51b0a 100644 --- a/BUILD +++ b/BUILD @@ -118,3 +118,35 @@ shell_command( output_directories=[".git/modules"], workdir="/", ) + +# this is used to quickly build a venv that includes all requirements and our wheels +for ic_name, ic in supported_python_interpreter_constraints().items(): + pex_binary( + name=f"st2-{ic_name}", + output_path=f"st2-{ic_name}.pex", + interpreter_constraints=ic, + dependencies=[ + # this should depend on all python_distribution targets + "st2actions", + "st2api", + "st2auth", + "st2client", + "st2common", + "st2reactor", + "st2stream", + "st2tests", + "contrib/runners/action_chain_runner", + "contrib/runners/announcement_runner", + "contrib/runners/http_runner", + "contrib/runners/inquirer_runner", + "contrib/runners/local_runner", + "contrib/runners/noop_runner", + "contrib/runners/orquesta_runner", + "contrib/runners/python_runner", + "contrib/runners/remote_runner", + "contrib/runners/winrm_runner", + ], + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=False, # always includes generated wheels, so transitive sources not needed + venv_hermetic_scripts=False, # do not add -sE to script shebangs + ) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 11131f20ee..d866a7f824 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -13,6 +13,16 @@ # limitations under the License. +# use this to parametrize targets as necessary. +# eg: interpreter_constraints=parametrize(**supported_python_interpreter_constraints) +def supported_python_interpreter_constraints(): + return dict( + py36=["CPython==3.6.*"], + py37=["CPython==3.7.*"], + py38=["CPython==3.8.*"], + ) + + def st2_publish_repos(): """Return the list of repos twine should publish to. diff --git a/pants.toml b/pants.toml index f6ca591fd6..a859e143ba 100644 --- a/pants.toml +++ b/pants.toml @@ -106,6 +106,8 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. +# Make sure to also update supported_interpreter_constraints in pants-plugins/macros.py +# (which is for use in parametrize() and similar ic-specific use-cases). st2_interpreter_constraints = "CPython>=3.8,<3.10" # This should match the pants interpreter_constraints: From 5d78ac623a187a20c6d8f8f4c5360a142a415d34 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 Apr 2023 22:08:53 -0500 Subject: [PATCH 02/60] move st2-py*.pex definition to packaging/BUILD --- BUILD | 32 -------------------------------- packaging/BUILD | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 32 deletions(-) create mode 100644 packaging/BUILD diff --git a/BUILD b/BUILD index 610df51b0a..f33988a645 100644 --- a/BUILD +++ b/BUILD @@ -118,35 +118,3 @@ shell_command( output_directories=[".git/modules"], workdir="/", ) - -# this is used to quickly build a venv that includes all requirements and our wheels -for ic_name, ic in supported_python_interpreter_constraints().items(): - pex_binary( - name=f"st2-{ic_name}", - output_path=f"st2-{ic_name}.pex", - interpreter_constraints=ic, - dependencies=[ - # this should depend on all python_distribution targets - "st2actions", - "st2api", - "st2auth", - "st2client", - "st2common", - "st2reactor", - "st2stream", - "st2tests", - "contrib/runners/action_chain_runner", - "contrib/runners/announcement_runner", - "contrib/runners/http_runner", - "contrib/runners/inquirer_runner", - "contrib/runners/local_runner", - "contrib/runners/noop_runner", - "contrib/runners/orquesta_runner", - "contrib/runners/python_runner", - "contrib/runners/remote_runner", - "contrib/runners/winrm_runner", - ], - include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=False, # always includes generated wheels, so transitive sources not needed - venv_hermetic_scripts=False, # do not add -sE to script shebangs - ) diff --git a/packaging/BUILD b/packaging/BUILD new file mode 100644 index 0000000000..d68d654f05 --- /dev/null +++ b/packaging/BUILD @@ -0,0 +1,32 @@ +# We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) +# that includes all requirements and our wheels. +for ic_name, ic in supported_python_interpreter_constraints().items(): + pex_binary( + name=f"st2-{ic_name}.pex", + output_path=f"st2-{ic_name}.pex", + interpreter_constraints=ic, + dependencies=[ + # this should depend on all python_distribution targets + "//st2actions", + "//st2api", + "//st2auth", + "//st2client", + "//st2common", + "//st2reactor", + "//st2stream", + "//st2tests", + "//contrib/runners/action_chain_runner", + "//contrib/runners/announcement_runner", + "//contrib/runners/http_runner", + "//contrib/runners/inquirer_runner", + "//contrib/runners/local_runner", + "//contrib/runners/noop_runner", + "//contrib/runners/orquesta_runner", + "//contrib/runners/python_runner", + "//contrib/runners/remote_runner", + "//contrib/runners/winrm_runner", + ], + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=False, # always includes generated wheels, so transitive sources not needed + venv_hermetic_scripts=False, # do not add -sE to script shebangs + ) From 26c49f3e325bdb8ab09db1ce13cf877131fcbd16 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 Apr 2023 00:34:33 -0500 Subject: [PATCH 03/60] copy packaging/*/systemd files from st2-packages.git --- packaging/common/systemd/st2api-generator | 56 +++++++++++++++++++ packaging/common/systemd/st2auth-generator | 56 +++++++++++++++++++ packaging/common/systemd/st2stream-generator | 56 +++++++++++++++++++ packaging/deb/systemd/st2actionrunner.service | 14 +++++ .../deb/systemd/st2actionrunner@.service | 21 +++++++ packaging/deb/systemd/st2api.service | 19 +++++++ packaging/deb/systemd/st2auth.service | 19 +++++++ .../deb/systemd/st2garbagecollector.service | 18 ++++++ packaging/deb/systemd/st2notifier.service | 18 ++++++ packaging/deb/systemd/st2rulesengine.service | 18 ++++++ packaging/deb/systemd/st2scheduler.service | 18 ++++++ .../deb/systemd/st2sensorcontainer.service | 18 ++++++ packaging/deb/systemd/st2stream.service | 19 +++++++ packaging/deb/systemd/st2timersengine.service | 18 ++++++ .../deb/systemd/st2workflowengine.service | 18 ++++++ packaging/rpm/systemd/st2actionrunner.service | 14 +++++ .../rpm/systemd/st2actionrunner@.service | 21 +++++++ packaging/rpm/systemd/st2api.service | 19 +++++++ packaging/rpm/systemd/st2auth.service | 19 +++++++ .../rpm/systemd/st2garbagecollector.service | 18 ++++++ packaging/rpm/systemd/st2notifier.service | 18 ++++++ packaging/rpm/systemd/st2rulesengine.service | 18 ++++++ packaging/rpm/systemd/st2scheduler.service | 18 ++++++ .../rpm/systemd/st2sensorcontainer.service | 18 ++++++ packaging/rpm/systemd/st2stream.service | 19 +++++++ packaging/rpm/systemd/st2timersengine.service | 18 ++++++ .../rpm/systemd/st2workflowengine.service | 18 ++++++ 27 files changed, 604 insertions(+) create mode 100755 packaging/common/systemd/st2api-generator create mode 100755 packaging/common/systemd/st2auth-generator create mode 100755 packaging/common/systemd/st2stream-generator create mode 100644 packaging/deb/systemd/st2actionrunner.service create mode 100644 packaging/deb/systemd/st2actionrunner@.service create mode 100644 packaging/deb/systemd/st2api.service create mode 100644 packaging/deb/systemd/st2auth.service create mode 100644 packaging/deb/systemd/st2garbagecollector.service create mode 100644 packaging/deb/systemd/st2notifier.service create mode 100644 packaging/deb/systemd/st2rulesengine.service create mode 100644 packaging/deb/systemd/st2scheduler.service create mode 100644 packaging/deb/systemd/st2sensorcontainer.service create mode 100644 packaging/deb/systemd/st2stream.service create mode 100644 packaging/deb/systemd/st2timersengine.service create mode 100644 packaging/deb/systemd/st2workflowengine.service create mode 100644 packaging/rpm/systemd/st2actionrunner.service create mode 100644 packaging/rpm/systemd/st2actionrunner@.service create mode 100644 packaging/rpm/systemd/st2api.service create mode 100644 packaging/rpm/systemd/st2auth.service create mode 100644 packaging/rpm/systemd/st2garbagecollector.service create mode 100644 packaging/rpm/systemd/st2notifier.service create mode 100644 packaging/rpm/systemd/st2rulesengine.service create mode 100644 packaging/rpm/systemd/st2scheduler.service create mode 100644 packaging/rpm/systemd/st2sensorcontainer.service create mode 100644 packaging/rpm/systemd/st2stream.service create mode 100644 packaging/rpm/systemd/st2timersengine.service create mode 100644 packaging/rpm/systemd/st2workflowengine.service diff --git a/packaging/common/systemd/st2api-generator b/packaging/common/systemd/st2api-generator new file mode 100755 index 0000000000..ae6d2e7eed --- /dev/null +++ b/packaging/common/systemd/st2api-generator @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import configparser +import logging +import time +import sys + +ST2SVC = "st2api" +DEFAULT_IP = "127.0.0.1" +DEFAULT_PORT = "9101" +ST2CFG = "/etc/st2/st2.conf" + +# Systemd passes 3 paths to a generator, normal_dir, early_dir, late_dir. +default_paths = ["/tmp", "/tmp", "/tmp"] +for i, p in enumerate(sys.argv[1:]): + default_paths[i] = p +EARLY_DIR, NORMAL_DIR, LATE_DIR = default_paths + +LOG_TO_DISK = True +LOG_KW = { + "level": logging.DEBUG, + "format": "%(asctime)s - %(levelname)s - %(message)s", +} +if LOG_TO_DISK: + LOG_KW["filename"] = f"{NORMAL_DIR}/{ST2SVC}_generator.log" + +logging.basicConfig(**LOG_KW) +LOG = logging.getLogger() + +LOG.debug( + f"Systemd directories: Early='{EARLY_DIR}' Normal='{NORMAL_DIR}' Late='{LATE_DIR}'" +) + +config = configparser.ConfigParser(strict=False) +config.read(ST2CFG) + +section = ST2SVC[3:] +bind_address = config[section].get("host", DEFAULT_IP) +bind_port = config[section].get("port", DEFAULT_PORT) + +contents = f"""[Unit] +# Generated by {sys.argv[0]} at {time.asctime(time.localtime())} +Description=StackStorm {ST2SVC} Socket. +PartOf={ST2SVC}.service +SourcePath={ST2CFG} + +[Socket] +ListenStream={bind_address}:{bind_port} + +[Install] +WantedBy=sockets.target +""" + +with open(f"{NORMAL_DIR}/{ST2SVC}.socket", "w") as f: + f.write(contents) + +LOG.info(f"{ST2SVC} generated.") diff --git a/packaging/common/systemd/st2auth-generator b/packaging/common/systemd/st2auth-generator new file mode 100755 index 0000000000..f1b017e99b --- /dev/null +++ b/packaging/common/systemd/st2auth-generator @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import configparser +import logging +import time +import sys + +ST2SVC="st2auth" +DEFAULT_IP="127.0.0.1" +DEFAULT_PORT="9100" +ST2CFG = "/etc/st2/st2.conf" + +# Systemd passes 3 paths to a generator, normal_dir, early_dir, late_dir. +default_paths = ["/tmp", "/tmp", "/tmp"] +for i, p in enumerate(sys.argv[1:]): + default_paths[i] = p +EARLY_DIR, NORMAL_DIR, LATE_DIR = default_paths + +LOG_TO_DISK = True +LOG_KW = { + "level": logging.DEBUG, + "format": "%(asctime)s - %(levelname)s - %(message)s", +} +if LOG_TO_DISK: + LOG_KW["filename"] = f"{NORMAL_DIR}/{ST2SVC}_generator.log" + +logging.basicConfig(**LOG_KW) +LOG = logging.getLogger() + +LOG.debug( + f"Systemd directories: Early='{EARLY_DIR}' Normal='{NORMAL_DIR}' Late='{LATE_DIR}'" +) + +config = configparser.ConfigParser(strict=False) +config.read(ST2CFG) + +section = ST2SVC[3:] +bind_address = config[section].get("host", DEFAULT_IP) +bind_port = config[section].get("port", DEFAULT_PORT) + +contents = f"""[Unit] +# Generated by {sys.argv[0]} at {time.asctime(time.localtime())} +Description=StackStorm {ST2SVC} Socket. +PartOf={ST2SVC}.service +SourcePath={ST2CFG} + +[Socket] +ListenStream={bind_address}:{bind_port} + +[Install] +WantedBy=sockets.target +""" + +with open(f"{NORMAL_DIR}/{ST2SVC}.socket", "w") as f: + f.write(contents) + +LOG.info(f"{ST2SVC} generated.") diff --git a/packaging/common/systemd/st2stream-generator b/packaging/common/systemd/st2stream-generator new file mode 100755 index 0000000000..e7d82e0f44 --- /dev/null +++ b/packaging/common/systemd/st2stream-generator @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import configparser +import logging +import time +import sys + +ST2SVC="st2stream" +DEFAULT_IP="127.0.0.1" +DEFAULT_PORT="9102" +ST2CFG = "/etc/st2/st2.conf" + +# Systemd passes 3 paths to a generator, normal_dir, early_dir, late_dir. +default_paths = ["/tmp", "/tmp", "/tmp"] +for i, p in enumerate(sys.argv[1:]): + default_paths[i] = p +EARLY_DIR, NORMAL_DIR, LATE_DIR = default_paths + +LOG_TO_DISK = True +LOG_KW = { + "level": logging.DEBUG, + "format": "%(asctime)s - %(levelname)s - %(message)s", +} +if LOG_TO_DISK: + LOG_KW["filename"] = f"{NORMAL_DIR}/{ST2SVC}_generator.log" + +logging.basicConfig(**LOG_KW) +LOG = logging.getLogger() + +LOG.debug( + f"Systemd directories: Early='{EARLY_DIR}' Normal='{NORMAL_DIR}' Late='{LATE_DIR}'" +) + +config = configparser.ConfigParser(strict=False) +config.read(ST2CFG) + +section = ST2SVC[3:] +bind_address = config[section].get("host", DEFAULT_IP) +bind_port = config[section].get("port", DEFAULT_PORT) + +contents = f"""[Unit] +# Generated by {sys.argv[0]} at {time.asctime(time.localtime())} +Description=StackStorm {ST2SVC} Socket. +PartOf={ST2SVC}.service +SourcePath={ST2CFG} + +[Socket] +ListenStream={bind_address}:{bind_port} + +[Install] +WantedBy=sockets.target +""" + +with open(f"{NORMAL_DIR}/{ST2SVC}.socket", "w") as f: + f.write(contents) + +LOG.info(f"{ST2SVC} generated.") diff --git a/packaging/deb/systemd/st2actionrunner.service b/packaging/deb/systemd/st2actionrunner.service new file mode 100644 index 0000000000..e0db64250b --- /dev/null +++ b/packaging/deb/systemd/st2actionrunner.service @@ -0,0 +1,14 @@ +[Unit] +Description=StackStorm service st2actionrunner +After=network.target + +[Service] +Type=oneshot +EnvironmentFile=-/etc/default/st2actionrunner +ExecStart=/bin/bash /opt/stackstorm/st2/bin/runners.sh start +ExecStop=/bin/bash /opt/stackstorm/st2/bin/runners.sh stop +PrivateTmp=true +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2actionrunner@.service b/packaging/deb/systemd/st2actionrunner@.service new file mode 100644 index 0000000000..f77a8707d4 --- /dev/null +++ b/packaging/deb/systemd/st2actionrunner@.service @@ -0,0 +1,21 @@ +[Unit] +Description=StackStorm service st2actionrunner +After=network.target +JoinsNamespaceOf=st2actionrunner.service + +[Service] +Type=simple +User=root +Group=st2packs +UMask=002 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +Environment="WORKERID=%i" +EnvironmentFile=-/etc/default/st2actionrunner +ExecStart=/opt/stackstorm/st2/bin/st2actionrunner $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2api.service b/packaging/deb/systemd/st2api.service new file mode 100644 index 0000000000..1921126db6 --- /dev/null +++ b/packaging/deb/systemd/st2api.service @@ -0,0 +1,19 @@ +[Unit] +Description=StackStorm service st2api +After=network.target st2api.socket +Requires=st2api.socket + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=-k eventlet -b 127.0.0.1:9101 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30 --log-config /etc/st2/logging.api.gunicorn.conf --error-logfile /var/log/st2/st2api.log" +EnvironmentFile=-/etc/default/st2api +ExecStart=/opt/stackstorm/st2/bin/gunicorn st2api.wsgi:application $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2auth.service b/packaging/deb/systemd/st2auth.service new file mode 100644 index 0000000000..54420192f8 --- /dev/null +++ b/packaging/deb/systemd/st2auth.service @@ -0,0 +1,19 @@ +[Unit] +Description=StackStorm service st2auth +After=network.target st2auth.socket +Requires=st2auth.socket + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=-k eventlet -b 127.0.0.1:9100 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30 --log-config /etc/st2/logging.auth.gunicorn.conf --error-logfile /var/log/st2/st2auth.log" +EnvironmentFile=-/etc/default/st2auth +ExecStart=/opt/stackstorm/st2/bin/gunicorn st2auth.wsgi:application $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2garbagecollector.service b/packaging/deb/systemd/st2garbagecollector.service new file mode 100644 index 0000000000..5d3061534a --- /dev/null +++ b/packaging/deb/systemd/st2garbagecollector.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2garbagecollector +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2garbagecollector +ExecStart=/opt/stackstorm/st2/bin/st2garbagecollector $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2notifier.service b/packaging/deb/systemd/st2notifier.service new file mode 100644 index 0000000000..31df3f3411 --- /dev/null +++ b/packaging/deb/systemd/st2notifier.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2notifier +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2notifier +ExecStart=/opt/stackstorm/st2/bin/st2notifier $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2rulesengine.service b/packaging/deb/systemd/st2rulesengine.service new file mode 100644 index 0000000000..5747154368 --- /dev/null +++ b/packaging/deb/systemd/st2rulesengine.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2rulesengine +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2rulesengine +ExecStart=/opt/stackstorm/st2/bin/st2rulesengine $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2scheduler.service b/packaging/deb/systemd/st2scheduler.service new file mode 100644 index 0000000000..ff87ab9e3a --- /dev/null +++ b/packaging/deb/systemd/st2scheduler.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2scheduler +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2scheduler +ExecStart=/opt/stackstorm/st2/bin/st2scheduler $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2sensorcontainer.service b/packaging/deb/systemd/st2sensorcontainer.service new file mode 100644 index 0000000000..67ec0898b6 --- /dev/null +++ b/packaging/deb/systemd/st2sensorcontainer.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2sensorcontainer +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2sensorcontainer +ExecStart=/opt/stackstorm/st2/bin/st2sensorcontainer $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2stream.service b/packaging/deb/systemd/st2stream.service new file mode 100644 index 0000000000..ade3ba5ddd --- /dev/null +++ b/packaging/deb/systemd/st2stream.service @@ -0,0 +1,19 @@ +[Unit] +Description=StackStorm service st2stream +After=network.target st2stream.socket +Requires=st2stream.socket + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=-k eventlet -b 127.0.0.1:9102 --workers 1 --threads 10 --graceful-timeout 10 --timeout 30 --log-config /etc/st2/logging.stream.gunicorn.conf --error-logfile /var/log/st2/st2stream.log" +EnvironmentFile=-/etc/default/st2stream +ExecStart=/opt/stackstorm/st2/bin/gunicorn st2stream.wsgi:application $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2timersengine.service b/packaging/deb/systemd/st2timersengine.service new file mode 100644 index 0000000000..768b07f30a --- /dev/null +++ b/packaging/deb/systemd/st2timersengine.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2timersengine +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2timersengine +ExecStart=/opt/stackstorm/st2/bin/st2timersengine $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/deb/systemd/st2workflowengine.service b/packaging/deb/systemd/st2workflowengine.service new file mode 100644 index 0000000000..5c9c96733a --- /dev/null +++ b/packaging/deb/systemd/st2workflowengine.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2workflowengine +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/default/st2workflowengine +ExecStart=/opt/stackstorm/st2/bin/st2workflowengine $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2actionrunner.service b/packaging/rpm/systemd/st2actionrunner.service new file mode 100644 index 0000000000..5d4f4e43e3 --- /dev/null +++ b/packaging/rpm/systemd/st2actionrunner.service @@ -0,0 +1,14 @@ +[Unit] +Description=StackStorm service st2actionrunner +After=network.target + +[Service] +Type=oneshot +EnvironmentFile=-/etc/sysconfig/st2actionrunner +ExecStart=/bin/bash /opt/stackstorm/st2/bin/runners.sh start +ExecStop=/bin/bash /opt/stackstorm/st2/bin/runners.sh stop +PrivateTmp=true +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2actionrunner@.service b/packaging/rpm/systemd/st2actionrunner@.service new file mode 100644 index 0000000000..2e1e971f41 --- /dev/null +++ b/packaging/rpm/systemd/st2actionrunner@.service @@ -0,0 +1,21 @@ +[Unit] +Description=StackStorm service st2actionrunner +After=network.target +JoinsNamespaceOf=st2actionrunner.service + +[Service] +Type=simple +User=root +Group=st2packs +UMask=002 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +Environment="WORKERID=%i" +EnvironmentFile=-/etc/sysconfig/st2actionrunner +ExecStart=/opt/stackstorm/st2/bin/st2actionrunner $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2api.service b/packaging/rpm/systemd/st2api.service new file mode 100644 index 0000000000..1b59b9b9c9 --- /dev/null +++ b/packaging/rpm/systemd/st2api.service @@ -0,0 +1,19 @@ +[Unit] +Description=StackStorm service st2api +After=network.target st2api.socket +Requires=st2api.socket + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=-k eventlet -b 127.0.0.1:9101 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30 --log-config /etc/st2/logging.api.gunicorn.conf --error-logfile /var/log/st2/st2api.log" +EnvironmentFile=-/etc/sysconfig/st2api +ExecStart=/opt/stackstorm/st2/bin/gunicorn st2api.wsgi:application $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2auth.service b/packaging/rpm/systemd/st2auth.service new file mode 100644 index 0000000000..260a785365 --- /dev/null +++ b/packaging/rpm/systemd/st2auth.service @@ -0,0 +1,19 @@ +[Unit] +Description=StackStorm service st2auth +After=network.target st2auth.socket +Requires=st2auth.socket + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=-k eventlet -b 127.0.0.1:9100 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30 --log-config /etc/st2/logging.auth.gunicorn.conf --error-logfile /var/log/st2/st2auth.log" +EnvironmentFile=-/etc/sysconfig/st2auth +ExecStart=/opt/stackstorm/st2/bin/gunicorn st2auth.wsgi:application $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2garbagecollector.service b/packaging/rpm/systemd/st2garbagecollector.service new file mode 100644 index 0000000000..62ff423930 --- /dev/null +++ b/packaging/rpm/systemd/st2garbagecollector.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2garbagecollector +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2garbagecollector +ExecStart=/opt/stackstorm/st2/bin/st2garbagecollector $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2notifier.service b/packaging/rpm/systemd/st2notifier.service new file mode 100644 index 0000000000..d0f098955d --- /dev/null +++ b/packaging/rpm/systemd/st2notifier.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2notifier +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2notifier +ExecStart=/opt/stackstorm/st2/bin/st2notifier $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2rulesengine.service b/packaging/rpm/systemd/st2rulesengine.service new file mode 100644 index 0000000000..c2ae7b0bee --- /dev/null +++ b/packaging/rpm/systemd/st2rulesengine.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2rulesengine +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2rulesengine +ExecStart=/opt/stackstorm/st2/bin/st2rulesengine $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2scheduler.service b/packaging/rpm/systemd/st2scheduler.service new file mode 100644 index 0000000000..725b46832b --- /dev/null +++ b/packaging/rpm/systemd/st2scheduler.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2scheduler +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2scheduler +ExecStart=/opt/stackstorm/st2/bin/st2scheduler $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2sensorcontainer.service b/packaging/rpm/systemd/st2sensorcontainer.service new file mode 100644 index 0000000000..d4f0674436 --- /dev/null +++ b/packaging/rpm/systemd/st2sensorcontainer.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2sensorcontainer +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2sensorcontainer +ExecStart=/opt/stackstorm/st2/bin/st2sensorcontainer $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2stream.service b/packaging/rpm/systemd/st2stream.service new file mode 100644 index 0000000000..742cc20384 --- /dev/null +++ b/packaging/rpm/systemd/st2stream.service @@ -0,0 +1,19 @@ +[Unit] +Description=StackStorm service st2stream +After=network.target st2stream.socket +Requires=st2stream.socket + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=-k eventlet -b 127.0.0.1:9102 --workers 1 --threads 10 --graceful-timeout 10 --timeout 30 --log-config /etc/st2/logging.stream.gunicorn.conf --error-logfile /var/log/st2/st2stream.log" +EnvironmentFile=-/etc/sysconfig/st2stream +ExecStart=/opt/stackstorm/st2/bin/gunicorn st2stream.wsgi:application $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2timersengine.service b/packaging/rpm/systemd/st2timersengine.service new file mode 100644 index 0000000000..f48606a710 --- /dev/null +++ b/packaging/rpm/systemd/st2timersengine.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2timersengine +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2timersengine +ExecStart=/opt/stackstorm/st2/bin/st2timersengine $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rpm/systemd/st2workflowengine.service b/packaging/rpm/systemd/st2workflowengine.service new file mode 100644 index 0000000000..537c20ddd9 --- /dev/null +++ b/packaging/rpm/systemd/st2workflowengine.service @@ -0,0 +1,18 @@ +[Unit] +Description=StackStorm service st2workflowengine +After=network.target + +[Service] +Type=simple +User=st2 +Group=st2 +Environment="DAEMON_ARGS=--config-file /etc/st2/st2.conf" +EnvironmentFile=-/etc/sysconfig/st2workflowengine +ExecStart=/opt/stackstorm/st2/bin/st2workflowengine $DAEMON_ARGS +TimeoutSec=60 +PrivateTmp=true +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target From 0b0b88bbe3dd37da6fdb39ceaca42f70ac77ec7c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 Apr 2023 00:39:11 -0500 Subject: [PATCH 04/60] copy st2-auth-pam requirement from st2-packages --- BUILD | 1 + requirements-pants.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/BUILD b/BUILD index f33988a645..93b85ebb85 100644 --- a/BUILD +++ b/BUILD @@ -65,6 +65,7 @@ target( dependencies=[ "//:reqs#st2-auth-backend-flat-file", "//:reqs#st2-auth-ldap", + "//:reqs#st2-auth-pam", ], ) diff --git a/requirements-pants.txt b/requirements-pants.txt index f0f269d6d5..2eef2492fc 100644 --- a/requirements-pants.txt +++ b/requirements-pants.txt @@ -78,6 +78,7 @@ stevedore # For backward compatibility reasons, flat file backend is installed by default st2-auth-backend-flat-file st2-auth-ldap @ git+https://github.com/StackStorm/st2-auth-ldap.git@master +st2-auth-pam @ git+https://github.com/StackStorm/st2-auth-pam.git@master st2-rbac-backend @ git+https://github.com/StackStorm/st2-rbac-backend.git@master # tabulate used by tools/log_watcher.py tabulate From 9acc51721b152665579babb9b1957012de740dbc Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 19 Dec 2024 18:44:32 -0600 Subject: [PATCH 05/60] Pants: Use **parametrize(...) groups in packaging/BUILD --- packaging/BUILD | 78 +++++++++++++++++++++++++---------------- pants-plugins/macros.py | 10 ------ pants.toml | 3 +- 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index d68d654f05..22af4dd342 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -1,32 +1,50 @@ # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) # that includes all requirements and our wheels. -for ic_name, ic in supported_python_interpreter_constraints().items(): - pex_binary( - name=f"st2-{ic_name}.pex", - output_path=f"st2-{ic_name}.pex", - interpreter_constraints=ic, - dependencies=[ - # this should depend on all python_distribution targets - "//st2actions", - "//st2api", - "//st2auth", - "//st2client", - "//st2common", - "//st2reactor", - "//st2stream", - "//st2tests", - "//contrib/runners/action_chain_runner", - "//contrib/runners/announcement_runner", - "//contrib/runners/http_runner", - "//contrib/runners/inquirer_runner", - "//contrib/runners/local_runner", - "//contrib/runners/noop_runner", - "//contrib/runners/orquesta_runner", - "//contrib/runners/python_runner", - "//contrib/runners/remote_runner", - "//contrib/runners/winrm_runner", - ], - include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=False, # always includes generated wheels, so transitive sources not needed - venv_hermetic_scripts=False, # do not add -sE to script shebangs - ) +pex_binary( + name="st2.pex", + dependencies=[ + # this should depend on all python_distribution targets + "//st2actions", + "//st2api", + "//st2auth", + "//st2client", + "//st2common", + "//st2reactor", + "//st2stream", + "//st2tests", + "//contrib/runners/action_chain_runner", + "//contrib/runners/announcement_runner", + "//contrib/runners/http_runner", + "//contrib/runners/inquirer_runner", + "//contrib/runners/local_runner", + "//contrib/runners/noop_runner", + "//contrib/runners/orquesta_runner", + "//contrib/runners/python_runner", + "//contrib/runners/remote_runner", + "//contrib/runners/winrm_runner", + ], + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=False, # always includes generated wheels, so transitive sources not needed + venv_hermetic_scripts=False, # do not add -sE to script shebangs + # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml + **parametrize( + "py38", + output_path="st2-py38.pex", + interpreter_constraints=["CPython==3.8.*"], + ), + **parametrize( + "py39", + output_path="st2-py39.pex", + interpreter_constraints=["CPython==3.9.*"], + ), + **parametrize( + "py310", + output_path="st2-py310.pex", + interpreter_constraints=["CPython==3.10.*"], + ), + **parametrize( + "py311", + output_path="st2-py311.pex", + interpreter_constraints=["CPython==3.11.*"], + ), +) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index d866a7f824..11131f20ee 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -13,16 +13,6 @@ # limitations under the License. -# use this to parametrize targets as necessary. -# eg: interpreter_constraints=parametrize(**supported_python_interpreter_constraints) -def supported_python_interpreter_constraints(): - return dict( - py36=["CPython==3.6.*"], - py37=["CPython==3.7.*"], - py38=["CPython==3.8.*"], - ) - - def st2_publish_repos(): """Return the list of repos twine should publish to. diff --git a/pants.toml b/pants.toml index a859e143ba..49f1410829 100644 --- a/pants.toml +++ b/pants.toml @@ -106,8 +106,7 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. -# Make sure to also update supported_interpreter_constraints in pants-plugins/macros.py -# (which is for use in parametrize() and similar ic-specific use-cases). +# On update, make sure to also update parametrizations in packaging/BUILD. st2_interpreter_constraints = "CPython>=3.8,<3.10" # This should match the pants interpreter_constraints: From c16a7789f473649bf0c529521c516dba686d3d79 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 20 Dec 2024 15:37:47 -0600 Subject: [PATCH 06/60] Packaging: add rpm/deb package metadata --- packaging/BUILD | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packaging/BUILD b/packaging/BUILD index 22af4dd342..60e83841c9 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -48,3 +48,50 @@ pex_binary( interpreter_constraints=["CPython==3.11.*"], ), ) + +# Relevant nFPM docs: +# - https://www.pantsbuild.org/stable/reference/targets/nfpm_deb_package +# - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package +# - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ + +_pkg_description = """ +StackStorm Event-driven automation +Package is full standalone st2 installation including all components +in a pre-built venv. +""" +_common_pkg_metadata = dict( + package_name="st2", + description=_pkg_description, + homepage="https://stackstorm.com", + license="Apache-2.0", + version="", # TODO: where does version come from? + # arch used to be "any", but that was not correct as the venv has compiled packages. + arch="amd64", # TODO: parametrize this? + platform="linux", +) +_maintainer = "StackStorm Engineering " # TODO: update this + +nfpm_deb_package( + name="st2.deb", + maintainer=_maintainer, + fields={ + # https://www.debian.org/doc/debian-policy/ch-controlfields.html#source + # "Source": "st2", # TODO: do we really have an "st2" source package? + # https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-vcs-fields + "Vcs-Git": "git://github.com/stackstorm/st2.git", + "Vcs-Browser": "https://github.com/stackstorm/st2", + }, + section="python", + priority="optional", + **_common_pkg_metadata, +) + +nfpm_rpm_package( + name="st2.rpm", + vendor="The StackStorm Project", + packager=_maintainer, + # group="System/Management", # was only useful for EL 5 and earlier + compression="zstd:default", # EL 9 + # compression="xz", # EL 8 + **_common_pkg_metadata, +) From cb346c3a3d3d68a8cce2edc4e2434fdec789f5d5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 20 Dec 2024 15:40:07 -0600 Subject: [PATCH 07/60] Packaging: stub pkg relationship metadata Includes stuff copied from control and spec files, as well as some details extracted from recently built rpm/deb files. --- packaging/BUILD | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/packaging/BUILD b/packaging/BUILD index 60e83841c9..391207e670 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -83,6 +83,41 @@ nfpm_deb_package( }, section="python", priority="optional", + # package relationships + predepends=[ + "dpkg (>= 1.16.16)", + # "${pre:Depends}", + "python3 (>= 3.8)" + # "${misc:Pre-Depends}", + "adduser", + ], + depends=[ + "sudo", + "adduser", + "libssl-dev", + "libffi-dev", + "git", + "libpam0g", + "openssh-server", + "openssh-client", + "bash", + "netbase", + # "${Depends}", + "python3-distutils", + "python3-dev", + # We use nFPM, not debhelpers, so these do not make sense: + # "${shlibs:Depends}", # We explicitly disabled the analogous RPM feature + "libc6 (>= 2.29)", + "libexpat1 (>= 2.1~beta3)", + "libgcc-s1 (>= 4.2)", + "libldap-2.4-2 (>= 2.4.11)", + "libstdc++6 (>= 5.2)", + "zlib1g (>= 1:1.2.0)", + # "${misc:Depends}", + ], + conflicts=[ + "st2common", + ], **_common_pkg_metadata, ) @@ -93,5 +128,77 @@ nfpm_rpm_package( # group="System/Management", # was only useful for EL 5 and earlier compression="zstd:default", # EL 9 # compression="xz", # EL 8 + # package relationships + depends=[ # aka Requires + "openssl-devel", + "libffi-devel", + "git", + "pam", + "openssh-server", + "openssh-clients", + "bash", + "setup", + # TODO: parametrize + "python3-devel", # EL 8 + "python38-devel", # EL 9 + # These are in the native built rpms (this list is deduplicated) + # "/bin/bash", + # "/bin/sh", + # "/opt/stackstorm/st2/bin/python", + # "/usr/bin/env", + # "config(st2) = 3.9dev-80", # provided by this package + # "ld-linux-x86-64.so.2()(64bit)", + # "ld-linux-x86-64.so.2(GLIBC_2.3)(64bit)", + # "libc.so.6()(64bit)", + # "libc.so.6(GLIBC_2.12)(64bit)", + # "libc.so.6(GLIBC_2.14)(64bit)", + # "libc.so.6(GLIBC_2.17)(64bit)", + # "libc.so.6(GLIBC_2.18)(64bit)", + # "libc.so.6(GLIBC_2.2.5)(64bit)", + # "libc.so.6(GLIBC_2.25)(64bit)", + # "libc.so.6(GLIBC_2.28)(64bit)", + # "libc.so.6(GLIBC_2.3)(64bit)", + # "libc.so.6(GLIBC_2.3.4)(64bit)", + # "libc.so.6(GLIBC_2.4)(64bit)", + # "libc.so.6(GLIBC_2.6)(64bit)", + # "libc.so.6(GLIBC_2.7)(64bit)", + # "libdl.so.2()(64bit)", + # "libdl.so.2(GLIBC_2.2.5)(64bit)", + # "libffi-devel", + # "libgcc_s.so.1()(64bit)", + # "libgcc_s.so.1(GCC_3.0)(64bit)", + # "libgcc_s.so.1(GCC_3.3)(64bit)", + # "libgcc_s.so.1(GCC_4.2.0)(64bit)", + # "liblber.so.2()(64bit)", + # "liblber.so.2(OPENLDAP_2.200)(64bit)", + # "libldap.so.2()(64bit)", + # "libldap.so.2(OPENLDAP_2.200)(64bit)", + # "libm.so.6()(64bit)", + # "libpthread.so.0()(64bit)", + # "libpthread.so.0(GLIBC_2.2.5)(64bit)", + # "libpthread.so.0(GLIBC_2.3.2)(64bit)", + # "libstdc++.so.6()(64bit)", + # "libstdc++.so.6(CXXABI_1.3)(64bit)", + # "libstdc++.so.6(CXXABI_1.3.7)(64bit)", + # "libstdc++.so.6(CXXABI_1.3.9)(64bit)", + # "libstdc++.so.6(GLIBCXX_3.4)(64bit)", + # "libstdc++.so.6(GLIBCXX_3.4.11)(64bit)", + # "libstdc++.so.6(GLIBCXX_3.4.21)(64bit)", + # "rpmlib(CompressedFileNames) = 3.0.4-1", + # "rpmlib(FileDigests) = 4.6.0-1", + # "rpmlib(PayloadFilesHavePrefix) = 4.0-1", + # "rpmlib(PayloadIsZstd) = 5.4.18-1", + # "rtld(GNU_HASH)", + ], + conflicts=[ + "st2common", + ], + # TODO: Does nFPM auto generate any provides entries? + # provides=[ + # # These are in the native built rpms + # "config(st2)", + # "st2", + # "st2(x86-64)", + # ], **_common_pkg_metadata, ) From 7a8f7cc5fe9829b7822d0f86ad44c2ed25e08dd7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 20 Dec 2024 15:43:53 -0600 Subject: [PATCH 08/60] Packaging: Include systemd files in deb/rpm packages --- packaging/BUILD | 8 ++++++++ packaging/common/systemd/BUILD | 20 ++++++++++++++++++++ packaging/deb/systemd/BUILD | 29 +++++++++++++++++++++++++++++ packaging/rpm/systemd/BUILD | 29 +++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 packaging/common/systemd/BUILD create mode 100644 packaging/deb/systemd/BUILD create mode 100644 packaging/rpm/systemd/BUILD diff --git a/packaging/BUILD b/packaging/BUILD index 391207e670..812d7e2fda 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -73,6 +73,10 @@ _maintainer = "StackStorm Engineering " # TODO: update nfpm_deb_package( name="st2.deb", + dependencies=[ + "./deb/systemd:services", + "./common/systemd:generators", + ], maintainer=_maintainer, fields={ # https://www.debian.org/doc/debian-policy/ch-controlfields.html#source @@ -123,6 +127,10 @@ nfpm_deb_package( nfpm_rpm_package( name="st2.rpm", + dependencies=[ + "./rpm/systemd:services", + "./common/systemd:generators", + ], vendor="The StackStorm Project", packager=_maintainer, # group="System/Management", # was only useful for EL 5 and earlier diff --git a/packaging/common/systemd/BUILD b/packaging/common/systemd/BUILD new file mode 100644 index 0000000000..946aac96b0 --- /dev/null +++ b/packaging/common/systemd/BUILD @@ -0,0 +1,20 @@ +_generators = ( + "st2api-generator", + "st2auth-generator", + "st2stream-generator", +) + +python_sources( + name="generator_files", + sources=_generators, +) + +nfpm_content_files( + name="generators", + description="systemd service generator files for nfpm packages", + dependencies=[":generator_files"], + files=[(gen, f"/usr/lib/systemd/system-generators/{gen}") for gen in _generators], + file_owner="root", + file_group="root", + file_mode="rwxr-xr-x", +) diff --git a/packaging/deb/systemd/BUILD b/packaging/deb/systemd/BUILD new file mode 100644 index 0000000000..620b07a635 --- /dev/null +++ b/packaging/deb/systemd/BUILD @@ -0,0 +1,29 @@ +_services = ( + "st2actionrunner.service", + "st2actionrunner@.service", + "st2api.service", + "st2auth.service", + "st2garbagecollector.service", + "st2notifier.service", + "st2rulesengine.service", + "st2scheduler.service", + "st2sensorcontainer.service", + "st2stream.service", + "st2timersengine.service", + "st2workflowengine.service", +) + +files( + name="service_files", + sources=_services, +) + +nfpm_content_files( + name="services", + description="systemd service files for nfpm deb packages", + dependencies=[":service_files"], + files=[(svc, f"/usr/lib/systemd/system/{svc}") for svc in _services], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/packaging/rpm/systemd/BUILD b/packaging/rpm/systemd/BUILD new file mode 100644 index 0000000000..2b2c823f93 --- /dev/null +++ b/packaging/rpm/systemd/BUILD @@ -0,0 +1,29 @@ +_services = ( + "st2actionrunner.service", + "st2actionrunner@.service", + "st2api.service", + "st2auth.service", + "st2garbagecollector.service", + "st2notifier.service", + "st2rulesengine.service", + "st2scheduler.service", + "st2sensorcontainer.service", + "st2stream.service", + "st2timersengine.service", + "st2workflowengine.service", +) + +files( + name="service_files", + sources=_services, +) + +nfpm_content_files( + name="services", + description="systemd service files for nfpm rpm packages", + dependencies=[":service_files"], + files=[(svc, f"/usr/lib/systemd/system/{svc}") for svc in _services], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) From a092fedcff7f99ea4ef4d226a63e851a404d1c44 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 20 Dec 2024 17:45:32 -0600 Subject: [PATCH 09/60] Packaging: Include rpm/deb script stubs These scripts need to be cleaned up, possibly by using some kind of templating to generate them. --- packaging/BUILD | 20 +++++ packaging/deb/scripts/BUILD | 0 packaging/deb/scripts/post-install.sh | 120 ++++++++++++++++++++++++++ packaging/deb/scripts/post-remove.sh | 92 ++++++++++++++++++++ packaging/deb/scripts/pre-install.sh | 114 ++++++++++++++++++++++++ packaging/deb/scripts/pre-remove.sh | 23 +++++ packaging/rpm/scripts/BUILD | 2 + packaging/rpm/scripts/post-install.sh | 31 +++++++ packaging/rpm/scripts/post-remove.sh | 32 +++++++ packaging/rpm/scripts/pre-install.sh | 60 +++++++++++++ packaging/rpm/scripts/pre-remove.sh | 27 ++++++ 11 files changed, 521 insertions(+) create mode 100644 packaging/deb/scripts/BUILD create mode 100644 packaging/deb/scripts/post-install.sh create mode 100644 packaging/deb/scripts/post-remove.sh create mode 100644 packaging/deb/scripts/pre-install.sh create mode 100644 packaging/deb/scripts/pre-remove.sh create mode 100644 packaging/rpm/scripts/BUILD create mode 100644 packaging/rpm/scripts/post-install.sh create mode 100644 packaging/rpm/scripts/post-remove.sh create mode 100644 packaging/rpm/scripts/pre-install.sh create mode 100644 packaging/rpm/scripts/pre-remove.sh diff --git a/packaging/BUILD b/packaging/BUILD index 812d7e2fda..7fdc38ca7e 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -74,9 +74,19 @@ _maintainer = "StackStorm Engineering " # TODO: update nfpm_deb_package( name="st2.deb", dependencies=[ + "./deb/scripts", "./deb/systemd:services", "./common/systemd:generators", ], + scripts=dict( + preinstall="deb/scripts/pre-install.sh", + postinstall="deb/scripts/post-install.sh", + preremove="deb/scripts/pre-remove.sh", + postremove="deb/scripts/post-remove.sh", + # config="", + # templates="", + # rules="", + ), maintainer=_maintainer, fields={ # https://www.debian.org/doc/debian-policy/ch-controlfields.html#source @@ -128,9 +138,19 @@ nfpm_deb_package( nfpm_rpm_package( name="st2.rpm", dependencies=[ + "./rpm/scripts", "./rpm/systemd:services", "./common/systemd:generators", ], + scripts=dict( + preinstall="rpm/scripts/pre-install.sh", + postinstall="rpm/scripts/post-install.sh", + preremove="rpm/scripts/pre-remove.sh", + postremove="rpm/scripts/post-remove.sh", + # pretrans="", + # posttrans="", + # verify="", + ), vendor="The StackStorm Project", packager=_maintainer, # group="System/Management", # was only useful for EL 5 and earlier diff --git a/packaging/deb/scripts/BUILD b/packaging/deb/scripts/BUILD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh new file mode 100644 index 0000000000..c17e4f120e --- /dev/null +++ b/packaging/deb/scripts/post-install.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# postinst script for st2 +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + +ST2_USER=st2 +PACKS_GROUP=st2packs +ST2_UPGRADESTAMP="/tmp/.stamp-stackstorm-st2-deb-package" +upgrading=0 + +_ST2_SERVICES=" +st2actionrunner.service +st2api.service +st2auth.service +st2garbagecollector.service +st2notifier.service +st2rulesengine.service +st2scheduler.service +st2sensorcontainer.service +st2stream.service +st2timersengine.service +st2workflowengine.service +" + +## Permissions of files which should be set on install +SET_PERMS=$(cat </dev/null 2>&1 || true + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# based on dh_systemd_enable/12.10ubuntu1 and dh_systemd_start/12.10ubuntu1 +if [ "$1" = "configure" ] || [ "$1" = "abort-upgrade" ] || [ "$1" = "abort-deconfigure" ] || [ "$1" = "abort-remove" ] ; then + for service in ${_ST2_SERVICES}; do + # This will only remove masks created by d-s-h on package removal. + deb-systemd-helper unmask "${service}" >/dev/null || true + + # was-enabled defaults to true, so new installations run enable. + if deb-systemd-helper --quiet was-enabled "${service}"; then + # Enables the unit on first installation, creates new + # symlinks on upgrades if the unit file has changed. + deb-systemd-helper enable "${service}" >/dev/null || true + else + # Update the statefile to add new symlinks (if any), which need to be + # cleaned up on purge. Also remove old symlinks. + deb-systemd-helper update-state "${service}" >/dev/null || true + fi + done + systemctl --system daemon-reload >/dev/null || true + if [ -n "$2" ]; then + _dh_action=restart + else + _dh_action=start + fi + deb-systemd-invoke $_dh_action ${_ST2_SERVICES} >/dev/null || true +fi + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. +# includes: +# dh_python2: dh-virtualenv +# dh_systemd_enable/12.10ubuntu1 +# dh_systemd_start/12.10ubuntu1 + +#DEBHELPER# + + + +exit 0 diff --git a/packaging/deb/scripts/post-remove.sh b/packaging/deb/scripts/post-remove.sh new file mode 100644 index 0000000000..8469d235d7 --- /dev/null +++ b/packaging/deb/scripts/post-remove.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# postrm script for st2 +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + +_ST2_SERVICES=" +st2actionrunner +st2api +st2auth +st2garbagecollector +st2notifier +st2rulesengine +st2scheduler +st2sensorcontainer +st2stream +st2timersengine +st2workflowengine +" + +## Save st2 logrotate config on remove, but wipe it out on purge. +preserve_logrotate() { + if [ "$1" = remove ]; then + [ -f /etc/logrotate.d/st2 ] && mv /etc/logrotate.d/st2-pkgsaved.disabled 1>/dev/null 2>&1 || : + elif [ "$1" = purge ]; then + rm -f /etc/logrotate.d/st2-pkgsaved.disabled 1>/dev/null 2>&1 || : + # Clean up other StackStorm related configs and directories + rm -rf /etc/st2 1>/dev/null 2>&1 || : + rm -rf /opt/stackstorm 1>/dev/null 2>&1 || : + rm -rf /root/.st2 1>/dev/null 2>&1 || : + rm -rf /var/log/st2 1>/dev/null 2>&1 || : + rm -f /etc/sudoers.d/st2 1>/dev/null 2>&1 || : + fi +} + +case "$1" in + remove|purge) + preserve_logrotate "$1" + ;; + upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# based on dh_systemd_start/12.10ubuntu1 +if [ -d /run/systemd/system ]; then + systemctl --system daemon-reload >/dev/null || true +fi + +for service in ${_ST2_SERVICES}; do + # based on dh_systemd_enable/12.10ubuntu1 and dh_systemd_start/12.10ubuntu1 + if [ "$1" = "remove" ]; then + if [ -x "/usr/bin/deb-systemd-helper" ]; then + deb-systemd-helper mask "${service}" >/dev/null || true + fi + fi + + if [ "$1" = "purge" ]; then + if [ -x "/usr/bin/deb-systemd-helper" ]; then + deb-systemd-helper purge "${service}" >/dev/null || true + deb-systemd-helper unmask "${service}" >/dev/null || true + fi + fi +done + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. +# includes: +# dh_python2: dh-virtualenv +# dh_systemd_enable/12.10ubuntu1 +# dh_systemd_start/12.10ubuntu1 + +#DEBHELPER# + +exit 0 diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh new file mode 100644 index 0000000000..daf0adc88d --- /dev/null +++ b/packaging/deb/scripts/pre-install.sh @@ -0,0 +1,114 @@ +#!/bin/sh +# preinst script for st2 +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `install' +# * `install' +# * `upgrade' +# * `abort-upgrade' +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + +PACKS_GROUP=st2packs +SYS_USER=stanley +ST2_USER=st2 +ST2_CONFPATH="/etc/st2/st2.conf" +ST2_UPGRADESTAMP="/tmp/.stamp-stackstorm-st2-deb-package" + +## Permissions of directories which has to be reset on upgrade +RESET_PERMS=$(cat </dev/null 2>&1) || + adduser --group --disabled-password --no-create-home --system $ST2_USER + + # make st2 member of st2packs group + (getent group $PACKS_GROUP 1>/dev/null 2>&1) || groupadd -r $PACKS_GROUP + (groups $ST2_USER 2>/dev/null | grep -q "\b${PACKS_GROUP}\b") || + usermod -a -G $PACKS_GROUP $ST2_USER + + # create stanley user (for actionrunner service) + if (! id $SYS_USER 1>/dev/null 2>&1); then + adduser --group $SYS_USER + adduser --disabled-password --gecos "" --ingroup $SYS_USER $SYS_USER + fi +} + +## [NOT USED!] Get current system user from the st2.conf +config_sysuser() { + # exit hooked + return 0 + local sysuser= + if [ -f $ST2_CONFPATH ]; then + sysuser=$(cat $ST2_CONFPATH | + sed -n -e '/\[system_user\]/,/\[.*\]\|\$/ { /\[.*\]/d; /user\s*=/ { s/\s*user\s*=\s*//; p } }') + fi + echo $sysuser +} + +## Update logrotate configuration +enable_logrotate() { + [ -f /etc/logrotate.d/st2-pkgsaved.disabled ] && + mv -f /etc/logrotate.d/st2-pkgsaved.disabled /etc/logrotate.d/st2 || : +} + +## Fix directories permissions on upgrade (different across maint scripts!) +# NB! USED FOR COMPATIBILITY ON UPGRADE FROM PREVIOUS VERSIONS OF PACKAGES. +# NB! In future package releases reseting permissions SHOULD BE REMOVED. +# +set_permissions() { + local fileperms="$1" mode= ownership= path= current_ownership= user= group= + + echo "$fileperms" | sed -e "s/_packsgroup/$PACKS_GROUP/g" -e "s/_st2user/$ST2_USER/g" | + while read mode ownership path; do + user=$(echo $ownership | cut -f1 -d:) + group=$(echo $ownership | cut -f2 -d:) + # set top level permissions whether it's a file or directory + [ -e $path ] || continue + chown $ownership $path && chmod $mode $path + + # recursively change permissions of children (since those are directories) + find $path -mindepth 1 -maxdepth 1 -not \( -user $user -group $group \) | + xargs -I {} sh -c "echo chown -R $ownership {} && echo chmod -R $mode {}" + done +} + +case "$1" in + install) + create_users + enable_logrotate + ;; + upgrade) + create_users + enable_logrotate + set_permissions "$RESET_PERMS" + touch $ST2_UPGRADESTAMP + ;; + abort-upgrade) + ;; + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packaging/deb/scripts/pre-remove.sh b/packaging/deb/scripts/pre-remove.sh new file mode 100644 index 0000000000..be1af9039e --- /dev/null +++ b/packaging/deb/scripts/pre-remove.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# prerm script for st2 + +set -e + +_ST2_SERVICES=" +st2actionrunner +st2api +st2auth +st2garbagecollector +st2notifier +st2rulesengine +st2scheduler +st2sensorcontainer +st2stream +st2timersengine +st2workflowengine +" + +# based on dh_systemd_start/12.10ubuntu1 +if [ -d /run/systemd/system ] && [ "$1" = remove ]; then + systemctl stop ${_ST2_SERVICES} >/dev/null || true +fi diff --git a/packaging/rpm/scripts/BUILD b/packaging/rpm/scripts/BUILD new file mode 100644 index 0000000000..9e055c1dc6 --- /dev/null +++ b/packaging/rpm/scripts/BUILD @@ -0,0 +1,2 @@ +# TODO: separate EL8 and EL9, or generate from templates somehow. +shell_sources() diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh new file mode 100644 index 0000000000..67d20480b9 --- /dev/null +++ b/packaging/rpm/scripts/post-install.sh @@ -0,0 +1,31 @@ +set -e + +_ST2_SERVICES=" +st2actionrunner +st2api +st2auth +st2garbagecollector +st2notifier +st2rulesengine +st2scheduler +st2sensorcontainer +st2stream +st2timersengine +st2workflowengine +" + +# EL 8: %service_post +if [ $1 -eq 1 ] ; then + # Initial installation + systemctl --no-reload preset ${_ST2_SERVICES} &>/dev/null || : +fi +# EL 9: %service_post +if [ $1 -eq 1 ] && [ -x "/usr/lib/systemd/systemd-update-helper" ]; then + # Initial installation + /usr/lib/systemd/systemd-update-helper install-system-units ${_ST2_SERVICES} || : +fi + +systemctl --no-reload enable ${_ST2_SERVICES} &>/dev/null || : + +# make sure that our socket generators run +systemctl daemon-reload &>/dev/null || : diff --git a/packaging/rpm/scripts/post-remove.sh b/packaging/rpm/scripts/post-remove.sh new file mode 100644 index 0000000000..0f08be40b3 --- /dev/null +++ b/packaging/rpm/scripts/post-remove.sh @@ -0,0 +1,32 @@ +set -e + +_ST2_SERVICES=" +st2actionrunner +st2actionrunner@ +st2api +st2auth +st2garbagecollector +st2notifier +st2rulesengine +st2scheduler +st2sensorcontainer +st2stream +st2timersengine +st2workflowengine +" + +# EL 8: %service_postun +if [ $1 -ge 1 ] ; then + # Package upgrade, not uninstall + systemctl try-restart ${_ST2_SERVICES} &>/dev/null || : +fi +# EL 9: %service_postun +if [ $1 -ge 1 ] && [ -x "/usr/lib/systemd/systemd-update-helper" ]; then + # Package upgrade, not uninstall + /usr/lib/systemd/systemd-update-helper mark-restart-system-units ${_ST2_SERVICES} || : +fi + +# Remove st2 logrotate config, since there's no analog of apt-get purge available +if [ $1 -eq 0 ]; then + rm -f /etc/logrotate.d/st2 +fi diff --git a/packaging/rpm/scripts/pre-install.sh b/packaging/rpm/scripts/pre-install.sh new file mode 100644 index 0000000000..45478a074b --- /dev/null +++ b/packaging/rpm/scripts/pre-install.sh @@ -0,0 +1,60 @@ +set -e + +PACKS_GROUP=st2packs +SYS_USER=stanley +ST2_USER=st2 + +## Permissions of directories which has to be reset on upgrade +RESET_PERMS=$(cat </dev/null 2>&1) || + adduser --no-create-home --system --user-group $ST2_USER + + # make st2 member of st2packs group + (getent group $PACKS_GROUP 1>/dev/null 2>&1) || groupadd -r $PACKS_GROUP + (groups $ST2_USER 2>/dev/null | grep -q "\b${PACKS_GROUP}\b") || + usermod -a -G $PACKS_GROUP $ST2_USER + + # create stanley user (unprivileged action user, we don't ship sudoers.d config) + (id $SYS_USER 1>/dev/null 2>&1) || + adduser --user-group $SYS_USER +} + +## Fix directories permissions on upgrade (different across maint scripts!) +# NB! USED FOR COMPATIBILITY ON UPGRADE FROM PREVIOUS VERSIONS OF PACKAGES. +# NB! In future package releases reseting permissions SHOULD BE REMOVED. +# +set_permissions() { + local fileperms="$1" mode= ownership= path= current_ownership= user= group= + + echo "$fileperms" | sed -e "s/_packsgroup/$PACKS_GROUP/g" -e "s/_st2user/$ST2_USER/g" | + while read mode ownership path; do + user=$(echo $ownership | cut -f1 -d:) + group=$(echo $ownership | cut -f2 -d:) + # set top level permissions whether it's a file or directory + [ -e $path ] || continue + chown $ownership $path && chmod $mode $path + + # recursively change permissions of children (since those are directories) + find $path -mindepth 1 -maxdepth 1 -not \( -user $user -group $group \) | + xargs -I {} sh -c "chown -R $ownership {} && chmod -R $mode {}" + done +} + +create_users + +# We perform upgrade (when install count > 1) +if [ "$1" -gt 1 ]; then + set_permissions "$RESET_PERMS" +fi diff --git a/packaging/rpm/scripts/pre-remove.sh b/packaging/rpm/scripts/pre-remove.sh new file mode 100644 index 0000000000..aeecd70401 --- /dev/null +++ b/packaging/rpm/scripts/pre-remove.sh @@ -0,0 +1,27 @@ +set -e + +_ST2_SERVICES=" +st2actionrunner +st2actionrunner@ +st2api +st2auth +st2garbagecollector +st2notifier +st2rulesengine +st2scheduler +st2sensorcontainer +st2stream +st2timersengine +st2workflowengine +" + +# EL 8: %service_preun +if [ $1 -eq 0 ] ; then + # Package removal, not upgrade + systemctl --no-reload disable --now ${_ST2_SERVICES} &>/dev/null || : +fi +# EL 9: %service_preun +if [ $1 -eq 0 ] && [ -x "/usr/lib/systemd/systemd-update-helper" ]; then + # Package removal, not upgrade + /usr/lib/systemd/systemd-update-helper remove-system-units ${_ST2_SERVICES} || : +fi From 1cfb45221b75cb88a76ce69cc515c19989e39d87 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 20 Dec 2024 17:47:14 -0600 Subject: [PATCH 10/60] Packaging: Include rpm/deb script stubs These scripts need to be cleaned up, possibly by using some kind of templating to generate them. --- packaging/deb/scripts/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/deb/scripts/BUILD b/packaging/deb/scripts/BUILD index e69de29bb2..3bd08979a8 100644 --- a/packaging/deb/scripts/BUILD +++ b/packaging/deb/scripts/BUILD @@ -0,0 +1,2 @@ +# TODO: maybe generate from templates somehow. +shell_sources() From 90c228c9ce889f7b310e4d088df9276b3103e8e1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 14:58:48 -0600 Subject: [PATCH 11/60] Packaging: Build /opt/stackstorm/st2 when running ./st2.pex Now we can treat st2.pex as a self-extracting venv installer. Use a preamble file, which pex executes before its bootstrap code, to make the pex just build /opt/stackstorm/st2 instead of teaching all the installers what a pex-tools are. --- packaging/BUILD | 12 +++++++++- packaging/pex_preamble.py | 48 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 packaging/pex_preamble.py diff --git a/packaging/BUILD b/packaging/BUILD index 7fdc38ca7e..a07b5147c3 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -1,8 +1,15 @@ +python_sources() + # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) # that includes all requirements and our wheels. pex_binary( name="st2.pex", + extra_build_args=[ + "--preamble-file", + f"source_files/{build_file_dir()}/pex_preamble.py", # preamble gets executed before the pex bootstrap + ], dependencies=[ + "./pex_preamble.py", # this should depend on all python_distribution targets "//st2actions", "//st2api", @@ -23,8 +30,11 @@ pex_binary( "//contrib/runners/remote_runner", "//contrib/runners/winrm_runner", ], + execution_mode="venv", + layout="zipapp", # zipapp creates a single file, loose and packed create directories + sh_boot=True, # faster startup time (only relevant for unpacking the pex) include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=False, # always includes generated wheels, so transitive sources not needed + include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml **parametrize( diff --git a/packaging/pex_preamble.py b/packaging/pex_preamble.py new file mode 100644 index 0000000000..c9ec720f3f --- /dev/null +++ b/packaging/pex_preamble.py @@ -0,0 +1,48 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +# This check makes re-exec safe by ensuring we modify env+argv once. +if os.environ.pop("ST2_PEX_EXTRACT", "0") not in ("1", "skip"): + os.environ["ST2_PEX_EXTRACT"] = "1" + + st2_config_path = os.environ.get("ST2_CONFIG_PATH", os.environ.get("ST2_CONF")) or "/etc/st2/st2.conf" + + # late import to minimize re-exec overhead (oslo_config is not available yet, so use stdlib here) + import configparser + + conf = configparser.ConfigParser() + conf.read_dict({"system": {"base_path": "/opt/stackstorm"}}) + conf.read(st2_config_path) + st2_base_path = conf.get("system", "base_path") + st2_base_path = os.environ.pop("ST2_SYSTEM__BASE_PATH", st2_base_path) + + st2_venv = os.path.join(st2_base_path, "st2") + if os.path.exists(st2_venv): + print(f"WARNING: This will overwrite {st2_venv}", file=sys.stderr) + + # This env var and sys.argv will create a venv in the st2_venv dir. + os.environ["PEX_TOOLS"] = "1" + sys.argv[1:1] = ( + "venv", + "--force", # remove and replace the venv if it exists + "--non-hermetic-scripts", # do not add -sE to python shebang + "--system-site-packages", + "--prompt=st2", + st2_venv, + ) + +# The standard PEX bootstrap code is below this line. From 346ac3a9c6bcf8d51ccd1b9bd559284519419efa Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 15:05:14 -0600 Subject: [PATCH 12/60] nfpm metadata cleanup --- packaging/BUILD | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index a07b5147c3..546cc48da6 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -73,7 +73,8 @@ _common_pkg_metadata = dict( package_name="st2", description=_pkg_description, homepage="https://stackstorm.com", - license="Apache-2.0", + # https://jfearn.fedorapeople.org/en-US/RPM/4/html-single/RPM_Guide/index.html#idp3030720 + license="Apache-2.0", # TODO: nFPM is putting this under Copyright tag instead of License version="", # TODO: where does version come from? # arch used to be "any", but that was not correct as the venv has compiled packages. arch="amd64", # TODO: parametrize this? @@ -111,7 +112,7 @@ nfpm_deb_package( predepends=[ "dpkg (>= 1.16.16)", # "${pre:Depends}", - "python3 (>= 3.8)" + "python3 (>= 3.8)", # "${misc:Pre-Depends}", "adduser", ], @@ -131,12 +132,12 @@ nfpm_deb_package( "python3-dev", # We use nFPM, not debhelpers, so these do not make sense: # "${shlibs:Depends}", # We explicitly disabled the analogous RPM feature - "libc6 (>= 2.29)", - "libexpat1 (>= 2.1~beta3)", - "libgcc-s1 (>= 4.2)", - "libldap-2.4-2 (>= 2.4.11)", - "libstdc++6 (>= 5.2)", - "zlib1g (>= 1:1.2.0)", + # "libc6 (>= 2.29)", + # "libexpat1 (>= 2.1~beta3)", + # "libgcc-s1 (>= 4.2)", + # "libldap-2.4-2 (>= 2.4.11)", + # "libstdc++6 (>= 5.2)", + # "zlib1g (>= 1:1.2.0)", # "${misc:Depends}", ], conflicts=[ From 133ab2d89f41624b418f8a0c64d2af31222376b0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 15:05:27 -0600 Subject: [PATCH 13/60] Pants: enable nFPM backend --- pants.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pants.toml b/pants.toml index 49f1410829..6a1f3b05f5 100644 --- a/pants.toml +++ b/pants.toml @@ -27,6 +27,9 @@ backend_packages = [ "pants.backend.shell", "pants.backend.shell.lint.shellcheck", + # packaging + "pants.backend.experimental.nfpm", + # internal plugins in pants-plugins/ "pants.backend.plugin_development", "api_spec", From ae7461c04b10cbe18f4d1834465d83bd5d0617f4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 15:06:32 -0600 Subject: [PATCH 14/60] Packaging: Add /opt/stackstorm/install to hold st2.pex and other archives --- packaging/BUILD | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packaging/BUILD b/packaging/BUILD index 546cc48da6..043149a238 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -64,6 +64,35 @@ pex_binary( # - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package # - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ +nfpm_content_file( + name="st2_venv", + description="Pex file that system packages can use to generate /opt/stackstorm/st2", + file_owner="root", + file_group="root", + file_mode="rwxr-x---", + dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex + **parametrize( + "py38", + dependencies=[":st2.pex@parametrize=py38"], + src="st2-py38.pex", + ), + **parametrize( + "py39", + dependencies=[":st2.pex@parametrize=py39"], + src="st2-py39.pex", + ), + **parametrize( + "py310", + dependencies=[":st2.pex@parametrize=py310"], + src="st2-py310.pex", + ), + **parametrize( + "py311", + dependencies=[":st2.pex@parametrize=py311"], + src="st2-py311.pex", + ), +) + _pkg_description = """ StackStorm Event-driven automation Package is full standalone st2 installation including all components @@ -88,6 +117,7 @@ nfpm_deb_package( "./deb/scripts", "./deb/systemd:services", "./common/systemd:generators", + ":st2_venv@parametrize=py38", ], scripts=dict( preinstall="deb/scripts/pre-install.sh", @@ -152,6 +182,7 @@ nfpm_rpm_package( "./rpm/scripts", "./rpm/systemd:services", "./common/systemd:generators", + ":st2_venv@parametrize=py38", ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", From d55e2a66af12a75af16250de382df5c2bda128bf Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 16:14:28 -0600 Subject: [PATCH 15/60] Pants: Move venv packaging targets to separate BUILD file --- packaging/BUILD | 88 -------------------------------------------- packaging/BUILD.venv | 87 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 88 deletions(-) create mode 100644 packaging/BUILD.venv diff --git a/packaging/BUILD b/packaging/BUILD index 043149a238..1514f151d1 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -1,98 +1,10 @@ python_sources() -# We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) -# that includes all requirements and our wheels. -pex_binary( - name="st2.pex", - extra_build_args=[ - "--preamble-file", - f"source_files/{build_file_dir()}/pex_preamble.py", # preamble gets executed before the pex bootstrap - ], - dependencies=[ - "./pex_preamble.py", - # this should depend on all python_distribution targets - "//st2actions", - "//st2api", - "//st2auth", - "//st2client", - "//st2common", - "//st2reactor", - "//st2stream", - "//st2tests", - "//contrib/runners/action_chain_runner", - "//contrib/runners/announcement_runner", - "//contrib/runners/http_runner", - "//contrib/runners/inquirer_runner", - "//contrib/runners/local_runner", - "//contrib/runners/noop_runner", - "//contrib/runners/orquesta_runner", - "//contrib/runners/python_runner", - "//contrib/runners/remote_runner", - "//contrib/runners/winrm_runner", - ], - execution_mode="venv", - layout="zipapp", # zipapp creates a single file, loose and packed create directories - sh_boot=True, # faster startup time (only relevant for unpacking the pex) - include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) - venv_hermetic_scripts=False, # do not add -sE to script shebangs - # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml - **parametrize( - "py38", - output_path="st2-py38.pex", - interpreter_constraints=["CPython==3.8.*"], - ), - **parametrize( - "py39", - output_path="st2-py39.pex", - interpreter_constraints=["CPython==3.9.*"], - ), - **parametrize( - "py310", - output_path="st2-py310.pex", - interpreter_constraints=["CPython==3.10.*"], - ), - **parametrize( - "py311", - output_path="st2-py311.pex", - interpreter_constraints=["CPython==3.11.*"], - ), -) - # Relevant nFPM docs: # - https://www.pantsbuild.org/stable/reference/targets/nfpm_deb_package # - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package # - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ -nfpm_content_file( - name="st2_venv", - description="Pex file that system packages can use to generate /opt/stackstorm/st2", - file_owner="root", - file_group="root", - file_mode="rwxr-x---", - dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex - **parametrize( - "py38", - dependencies=[":st2.pex@parametrize=py38"], - src="st2-py38.pex", - ), - **parametrize( - "py39", - dependencies=[":st2.pex@parametrize=py39"], - src="st2-py39.pex", - ), - **parametrize( - "py310", - dependencies=[":st2.pex@parametrize=py310"], - src="st2-py310.pex", - ), - **parametrize( - "py311", - dependencies=[":st2.pex@parametrize=py311"], - src="st2-py311.pex", - ), -) - _pkg_description = """ StackStorm Event-driven automation Package is full standalone st2 installation including all components diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv new file mode 100644 index 0000000000..1232396b32 --- /dev/null +++ b/packaging/BUILD.venv @@ -0,0 +1,87 @@ +# We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) +# that includes all requirements and our wheels. +pex_binary( + name="st2.pex", + extra_build_args=[ + "--preamble-file", + f"source_files/{build_file_dir()}/pex_preamble.py", # preamble gets executed before the pex bootstrap + ], + dependencies=[ + "./pex_preamble.py", + # this should depend on all python_distribution targets + "//st2actions", + "//st2api", + "//st2auth", + "//st2client", + "//st2common", + "//st2reactor", + "//st2stream", + "//st2tests", + "//contrib/runners/action_chain_runner", + "//contrib/runners/announcement_runner", + "//contrib/runners/http_runner", + "//contrib/runners/inquirer_runner", + "//contrib/runners/local_runner", + "//contrib/runners/noop_runner", + "//contrib/runners/orquesta_runner", + "//contrib/runners/python_runner", + "//contrib/runners/remote_runner", + "//contrib/runners/winrm_runner", + ], + execution_mode="venv", + layout="zipapp", # zipapp creates a single file, loose and packed create directories + sh_boot=True, # faster startup time (only relevant for unpacking the pex) + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) + venv_hermetic_scripts=False, # do not add -sE to script shebangs + # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml + **parametrize( + "py38", + output_path="st2-py38.pex", + interpreter_constraints=["CPython==3.8.*"], + ), + **parametrize( + "py39", + output_path="st2-py39.pex", + interpreter_constraints=["CPython==3.9.*"], + ), + **parametrize( + "py310", + output_path="st2-py310.pex", + interpreter_constraints=["CPython==3.10.*"], + ), + **parametrize( + "py311", + output_path="st2-py311.pex", + interpreter_constraints=["CPython==3.11.*"], + ), +) + +nfpm_content_file( + name="st2_venv", + description="Pex file that system packages can use to generate /opt/stackstorm/st2", + file_owner="root", + file_group="root", + file_mode="rwxr-x---", + dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex + **parametrize( + "py38", + dependencies=[":st2.pex@parametrize=py38"], + src="st2-py38.pex", + ), + **parametrize( + "py39", + dependencies=[":st2.pex@parametrize=py39"], + src="st2-py39.pex", + ), + **parametrize( + "py310", + dependencies=[":st2.pex@parametrize=py310"], + src="st2-py310.pex", + ), + **parametrize( + "py311", + dependencies=[":st2.pex@parametrize=py311"], + src="st2-py311.pex", + ), +) From 5bbae5960ad8bb54d590205757ee282a2b9da28c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:28:01 -0600 Subject: [PATCH 16/60] pants: Dry parametrization in packaging/BUILD.venv --- packaging/BUILD.venv | 66 +++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index 1232396b32..6910dd0584 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -1,5 +1,14 @@ # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) # that includes all requirements and our wheels. + +def _pex_py3(minor: str): + return parametrize( + f"py3{minor}", + output_path=f"${{spec_path_normalized}}/st2-py3{minor}.pex", + interpreter_constraints=[f"CPython==3.{minor}.*"], + ) + + pex_binary( name="st2.pex", extra_build_args=[ @@ -35,28 +44,21 @@ pex_binary( include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml - **parametrize( - "py38", - output_path="st2-py38.pex", - interpreter_constraints=["CPython==3.8.*"], - ), - **parametrize( - "py39", - output_path="st2-py39.pex", - interpreter_constraints=["CPython==3.9.*"], - ), - **parametrize( - "py310", - output_path="st2-py310.pex", - interpreter_constraints=["CPython==3.10.*"], - ), - **parametrize( - "py311", - output_path="st2-py311.pex", - interpreter_constraints=["CPython==3.11.*"], - ), + **_pex_py3("8"), + **_pex_py3("9"), + # **_pex_py3("10"), + # **_pex_py3("11"), ) + +def _venv_py3(minor: str): + return parametrize( + f"py3{minor}", + dependencies=[f":st2.pex@parametrize=py3{minor}"], + src=f"st2-py3{minor}.pex", # relative to this BUILD file + ) + + nfpm_content_file( name="st2_venv", description="Pex file that system packages can use to generate /opt/stackstorm/st2", @@ -64,24 +66,8 @@ nfpm_content_file( file_group="root", file_mode="rwxr-x---", dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex - **parametrize( - "py38", - dependencies=[":st2.pex@parametrize=py38"], - src="st2-py38.pex", - ), - **parametrize( - "py39", - dependencies=[":st2.pex@parametrize=py39"], - src="st2-py39.pex", - ), - **parametrize( - "py310", - dependencies=[":st2.pex@parametrize=py310"], - src="st2-py310.pex", - ), - **parametrize( - "py311", - dependencies=[":st2.pex@parametrize=py311"], - src="st2-py311.pex", - ), + **_venv_py3("8"), + **_venv_py3("9"), + # **_venv_py3("10"), + # **_venv_py3("11"), ) From 34af710727876b537406c8f310a922ca030a1316 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:36:00 -0600 Subject: [PATCH 17/60] Packaging: Add self-install archives for each pack to /opt/stackstorm/install --- contrib/chatops/BUILD | 8 +++ contrib/chatops/tests/BUILD | 2 +- contrib/core/BUILD | 11 ++++ contrib/core/tests/BUILD | 2 +- contrib/debug/BUILD | 7 +++ contrib/default/BUILD | 6 +++ contrib/examples/BUILD | 26 ++++++++++ contrib/examples/actions/windows/BUILD | 3 ++ contrib/examples/tests/BUILD | 2 +- contrib/hello_st2/BUILD | 16 ++++++ contrib/linux/BUILD | 12 +++++ contrib/linux/tests/BUILD | 2 +- contrib/packs/BUILD | 9 ++++ contrib/packs/tests/BUILD | 2 +- packaging/BUILD | 4 ++ packaging/BUILD.packs | 29 +++++++++++ pants-plugins/macros.py | 71 ++++++++++++++++++++++++++ pants.toml | 3 +- 18 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 contrib/examples/actions/windows/BUILD create mode 100644 packaging/BUILD.packs diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 888be3a426..c9f2a2d183 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -3,3 +3,11 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + "./tests", + ], +) diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index ead8561daa..63c5ba6b03 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 9df7a372c9..aa5dd8555c 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -10,9 +10,20 @@ python_requirements( ) python_sources( + # this is for fixture.py dependencies=[ ":metadata", "./actions", "./actions/send_mail:send_mail_resources", ], ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions/send_mail:send_mail_resources", + "./tests", + ], +) diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index 1a4583beb6..c84e095e16 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 888be3a426..4713a9b1cc 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -3,3 +3,10 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + ], +) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 888be3a426..fc15b27bea 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -3,3 +3,9 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ], +) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index ab10cd1c85..22d0401478 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -18,3 +18,29 @@ python_requirement( name="flask", requirements=["flask"], ) + +# Also capture the requirements file for distribution in the pack archive. +files( + name="reqs", + sources=["requirements*.txt"], +) + +st2_pack_archive( + # we need to list targets of all files because transitive dep targets are NOT included + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions:shell", + "./actions/bash_exit_code", + "./actions/bash_ping", + "./actions/bash_random", + "./actions/pythonactions", + "./actions/ubuntu_pkg_info", + "./actions/ubuntu_pkg_info/lib", + "./actions/windows", + "./lib", + "./sensors", + "./tests", + ], +) diff --git a/contrib/examples/actions/windows/BUILD b/contrib/examples/actions/windows/BUILD new file mode 100644 index 0000000000..c6a0c2c7d9 --- /dev/null +++ b/contrib/examples/actions/windows/BUILD @@ -0,0 +1,3 @@ +files( + sources=["*.ps1"], +) diff --git a/contrib/examples/tests/BUILD b/contrib/examples/tests/BUILD index 25a2e7cc4b..0ddb50de63 100644 --- a/contrib/examples/tests/BUILD +++ b/contrib/examples/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 888be3a426..6938233485 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -3,3 +3,19 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +# Capture the requirements file for distribution in the pack archive; +# we do not need to use `python_requirements()` for this sample file. +files( + name="reqs", + sources=["requirements*.txt"], +) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./sensors", + ], +) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 201435eecc..e4619b05cb 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -7,3 +7,15 @@ pack_metadata( python_requirements( name="reqs", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions/checks", + # "./actions/lib", # nothing in dir + "./sensors", + "./tests", + ], +) diff --git a/contrib/linux/tests/BUILD b/contrib/linux/tests/BUILD index 25a2e7cc4b..0ddb50de63 100644 --- a/contrib/linux/tests/BUILD +++ b/contrib/linux/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 888be3a426..cb69238387 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -3,3 +3,12 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + "./actions/pack_mgmt", + "./tests", + ], +) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index c8265214ca..6326aced98 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/packaging/BUILD b/packaging/BUILD index 1514f151d1..93d3db54ed 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -30,6 +30,8 @@ nfpm_deb_package( "./deb/systemd:services", "./common/systemd:generators", ":st2_venv@parametrize=py38", + ":default_packs", + ":extra_packs", # TODO: Should we include these even though they won't be installed? ], scripts=dict( preinstall="deb/scripts/pre-install.sh", @@ -95,6 +97,8 @@ nfpm_rpm_package( "./rpm/systemd:services", "./common/systemd:generators", ":st2_venv@parametrize=py38", + ":default_packs", + ":extra_packs", # TODO: Should we include these even though they won't be installed? ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", diff --git a/packaging/BUILD.packs b/packaging/BUILD.packs new file mode 100644 index 0000000000..b1b2f97af3 --- /dev/null +++ b/packaging/BUILD.packs @@ -0,0 +1,29 @@ +# These are packs that should be installed with the default ST2 install. +_DEFAULT_PACKS=( + "chatops", + "core", + "default", + "linux", + "packs", +) + +# These are packs that should NOT be installed with the default ST2 install. +_EXTRA_PACKS=( + "debug", + "examples", + "hello_st2", +) + +# :archive_for_npfm targets created by st2_pack_archive() macro (see pants-plugins/macros.py) +target( + name="default_packs", + dependencies=[ + f"//contrib/{pack}:archive_for_nfpm" for pack in _DEFAULT_PACKS + ], +) +target( + name="extra_packs", + dependencies=[ + f"//contrib/{pack}:archive_for_nfpm" for pack in _EXTRA_PACKS + ], +) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 11131f20ee..0381f20d6f 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -112,6 +112,77 @@ def st2_component_python_distribution(**kwargs): python_distribution(**kwargs) # noqa: F821 +# Default copied from PEX (which uses zipfile standard MS-dOS epoch). +# https://github.com/pex-tool/pex/blob/v2.1.137/pex/common.py#L39-L45 +MTIME = "1980-01-01T00:00:00Z" + + +def st2_pack_archive(**kwargs): + """Create a makeself_archive using files from the given dependencies. + + This macro should be used in the same BUILD file as the pack_metadata target. + """ + build_file_path = build_file_dir() # noqa: F821 + if "st2tests" == build_file_path.parts[0]: + # avoid creating duplicate archive for the core pack + # which is also located under st2tests/st2tests/fixtures/packs + return + pack_name = build_file_path.name # noqa: F821 + + dependencies = kwargs.pop("dependencies", []) + if ":metadata" not in dependencies: + dependencies = [":metadata", *dependencies] + + # This is basically a "wrap_as_files" target (which does not exist yet) + shell_command( # noqa: F821 + name="files", + execution_dependencies=dependencies, + command="true", + output_directories=["."], + root_output_directory=".", + ) + + makeself_archive( # noqa: F821 + name="archive", + label=f"{pack_name} StackStorm pack", + files=[ + ":files", # archive contents + "//:license", # LICENSE file included in archive header, excluded from contents + ], + # startup_script=["echo", "pack-archive"], + args=( + # Makeself expects '--arg value' (space) not '--arg=value' (equals) for cmdline + "--license", + "__archive/LICENSE", + "--target", + f"/opt/stackstorm/packs/{pack_name}", + # reproducibility flags: + "--tar-extra", # extra tar args: '--arg=value' (equals delimited) space separated + f"--owner=root:0 --group=root:0 --mtime={MTIME} --exclude=LICENSE", # TODO: include LICENSE file? + "--packaging-date", + MTIME, # TODO: maybe use release date instead of an epoch date? + # compression/encryption flags: + # "--gzip", # gzip is the default compressor + # "--complevel", "9", # 9 is the default compression level + # "--gpg-encrypt", # gpg (encrypt only) handles compression if selected + # "--gpg-asymmetric-encrypt-sign", # gpg (encrypt and sign) handles compression if selected + # "--gpg-extra", "...", # if using gpg, pass extra gpg args here + # "--nocomp", # maybe use no compression to use rpm/deb's compression instead of gzip (default)? + ), + output_path=f"packaging/packs/{pack_name}.tgz.run", + ) + + nfpm_content_file( # noqa: F821 + name=f"archive_for_nfpm", + dependencies=[":archive"], + src=f"packaging/packs/{pack_name}.tgz.run", + dst=f"/opt/stackstorm/install/packs/{pack_name}.tgz.run", + file_owner="root", + file_group="root", + file_mode="rwxr-x---", + ) + + def st2_shell_sources_and_resources(**kwargs): """This creates a shell_sources and a resources target. diff --git a/pants.toml b/pants.toml index 6a1f3b05f5..e4c496f0c0 100644 --- a/pants.toml +++ b/pants.toml @@ -28,6 +28,7 @@ backend_packages = [ "pants.backend.shell.lint.shellcheck", # packaging + "pants.backend.experimental.makeself", "pants.backend.experimental.nfpm", # internal plugins in pants-plugins/ @@ -46,8 +47,6 @@ pants_ignore.add = [ "test_dist_utils.py", "setup.py", # keep tailor from using legacy requirements files (not for pants) - "contrib/examples/requirements.txt", - "contrib/hello_st2/requirements.txt", "contrib/runners/*/in-requirements.txt", "contrib/runners/*/requirements.txt", "st2*/in-requirements.txt", From a52837bb088e88ae9c516fd9fa9de43ca72a4329 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:36:48 -0600 Subject: [PATCH 18/60] Packaging: Add common dirs to rpm/deb --- packaging/BUILD | 2 ++ packaging/common/BUILD | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 packaging/common/BUILD diff --git a/packaging/BUILD b/packaging/BUILD index 93d3db54ed..a76143509d 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -29,6 +29,7 @@ nfpm_deb_package( "./deb/scripts", "./deb/systemd:services", "./common/systemd:generators", + "./common:dirs", ":st2_venv@parametrize=py38", ":default_packs", ":extra_packs", # TODO: Should we include these even though they won't be installed? @@ -96,6 +97,7 @@ nfpm_rpm_package( "./rpm/scripts", "./rpm/systemd:services", "./common/systemd:generators", + "./common:dirs", ":st2_venv@parametrize=py38", ":default_packs", ":extra_packs", # TODO: Should we include these even though they won't be installed? diff --git a/packaging/common/BUILD b/packaging/common/BUILD new file mode 100644 index 0000000000..ed07f1f058 --- /dev/null +++ b/packaging/common/BUILD @@ -0,0 +1,18 @@ +nfpm_content_dirs( + name="dirs", + dirs=[ + "/etc/st2", + "/opt/stackstorm/configs", + "/opt/stackstorm/exports", + "/opt/stackstorm/overrides", + "/opt/stackstorm/packs", + "/opt/stackstorm/st2", + "/opt/stackstorm/virtualenvs", + "/usr/share/doc/st2", + "/var/log/st2", + "/var/run/st2", + ], + file_owner="root", + file_group="root", + file_mode="rwxr-xr-x", +) From d51eccebd4b15211e19c7758a27999d519299e77 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:39:18 -0600 Subject: [PATCH 19/60] pants-plugins/pack_metadata: include Markdown files --- .../pack_metadata/python_rules/python_pack_content_test.py | 1 + pants-plugins/pack_metadata/target_types.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index 33c1389bb3..cf25605b71 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -36,6 +36,7 @@ ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.pack_doc,), 1, ".md"), ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), ((PackContentResourceTypes.rule_metadata,), 0, ""), diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 01d80c24ad..8fa0c8843b 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -43,6 +43,7 @@ class PackContentResourceTypes(Enum): pack_config_schema = "pack_config_schema" pack_config_example = "pack_config_example" pack_icon = "pack_icon" + pack_doc = "pack_doc" # in subdirectory (see _content_type_by_path_parts below action_metadata = "action_metadata" action_chain_workflow = "action_chain_workflow" @@ -86,6 +87,8 @@ def compute_value( if value is not None: return PackContentResourceTypes(value) path = PurePath(address.relative_file_path) + if path.suffix == ".md": + return PackContentResourceTypes.pack_doc _yaml_suffixes = (".yaml", ".yml") if len(path.parent.parts) == 0: # in the pack root @@ -123,8 +126,7 @@ class PackMetadataSourcesField(ResourcesGeneratingSourcesField): "**/*.yml", "icon.png", # used in st2web ui # "requirements*.txt", # including this causes target conflicts - # "README.md", - # "HISTORY.md", + "**/*.md", # including README.md, HISTORY.md # exclude yaml files under tests "!tests/**/*.yml", "!tests/**/*.yaml", From 4b5dcdf87588b803ffdcff236e50cf4d79956421 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 20:10:38 -0600 Subject: [PATCH 20/60] Packaging: dir permissions --- packaging/common/BUILD | 62 ++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/packaging/common/BUILD b/packaging/common/BUILD index ed07f1f058..d9e4c5bdd0 100644 --- a/packaging/common/BUILD +++ b/packaging/common/BUILD @@ -1,18 +1,52 @@ +_packs_group = "st2packs" +_st2_user = "st2" + nfpm_content_dirs( - name="dirs", - dirs=[ - "/etc/st2", - "/opt/stackstorm/configs", - "/opt/stackstorm/exports", - "/opt/stackstorm/overrides", - "/opt/stackstorm/packs", - "/opt/stackstorm/st2", - "/opt/stackstorm/virtualenvs", - "/usr/share/doc/st2", - "/var/log/st2", - "/var/run/st2", - ], - file_owner="root", + name="root_dirs", + dirs=[ + # "/etc/bash_completion.d", + # "/etc/logrotate.d", + "/etc/st2", + "/opt/stackstorm", + "/opt/stackstorm/st2", + "/usr/share/doc/st2", + "/var/log/st2", + "/var/run/st2", + ], + file_owner="root", file_group="root", file_mode="rwxr-xr-x", ) + +nfpm_content_dirs( + name="packs_group_dirs", + dirs=[ + "/opt/stackstorm/packs", + "/opt/stackstorm/virtualenvs", + "/usr/share/doc/st2/examples", + ], + file_owner="root", + file_group=_packs_group, + file_mode="rwxrwxr-x", +) + +nfpm_content_dirs( + name="st2_user_dirs", + dirs=[ + "/opt/stackstorm/configs", + "/opt/stackstorm/exports", + "/opt/stackstorm/overrides", + ], + file_owner=_st2_user, + file_group="root", + file_mode="rwxr-xr-x", +) + +target( + name="dirs", + dependencies=[ + ":root_dirs", + ":packs_group_dirs", + ":st2_user_dirs", + ], +) From f366706b5db0b7ab9f0ac9ff88ed89522b0f0fd7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 20:11:30 -0600 Subject: [PATCH 21/60] Packaging: add global cmd symlinks --- packaging/BUILD | 2 ++ packaging/common/BUILD | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/packaging/BUILD b/packaging/BUILD index a76143509d..2815cfa478 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -30,6 +30,7 @@ nfpm_deb_package( "./deb/systemd:services", "./common/systemd:generators", "./common:dirs", + "./common:symlinks", ":st2_venv@parametrize=py38", ":default_packs", ":extra_packs", # TODO: Should we include these even though they won't be installed? @@ -98,6 +99,7 @@ nfpm_rpm_package( "./rpm/systemd:services", "./common/systemd:generators", "./common:dirs", + "./common:symlinks", ":st2_venv@parametrize=py38", ":default_packs", ":extra_packs", # TODO: Should we include these even though they won't be installed? diff --git a/packaging/common/BUILD b/packaging/common/BUILD index d9e4c5bdd0..a9b3c0aef7 100644 --- a/packaging/common/BUILD +++ b/packaging/common/BUILD @@ -50,3 +50,27 @@ target( ":st2_user_dirs", ], ) + +nfpm_content_symlinks( + name="symlinks", + symlinks=[ + # the symlink src will be made available during post-install script + (f"/opt/stackstorm/st2/bin/{cmd}", f"/usr/bin/{cmd}") + for cmd in ( + "st2", + "st2-trigger-refire", + "st2-rule-tester", + "st2-run-pack-tests", + "st2-register-content", + "st2-apply-rbac-definitions", + "st2-bootstrap-rmq", + "st2-generate-symmetric-crypto-key", + "st2-self-check", + "st2-track-result", + "st2-validate-pack-config", + "st2ctl", + ) + ], + file_owner="root", + file_group="root", +) From 8d6a14f37896205b1a91cefdf49dbafd1c9db08f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 21:50:08 -0600 Subject: [PATCH 22/60] Packaging: add pants metadata for conf files --- conf/BUILD | 15 +++++++++++++++ packaging/BUILD | 2 ++ packaging/common/BUILD | 13 +++++++++++++ st2actions/conf/BUILD | 21 +++++++++++++++++++++ st2api/conf/BUILD | 17 +++++++++++++++++ st2auth/conf/BUILD | 17 +++++++++++++++++ st2client/conf/BUILD | 10 ++++++++++ st2reactor/conf/BUILD | 21 +++++++++++++++++++++ st2stream/conf/BUILD | 17 +++++++++++++++++ 9 files changed, 133 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index a3374bfca3..0bc4015eee 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -57,3 +57,18 @@ file( name="st2_package_conf", source="st2.package.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":st2_package_conf", + ":logrotate", + ], + files=[ + ("st2.package.conf", "/etc/st2/st2.conf"), + ("logrotate.conf", "/etc/logrotate.d/st2"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/packaging/BUILD b/packaging/BUILD index 2815cfa478..99c530bd3b 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -29,6 +29,7 @@ nfpm_deb_package( "./deb/scripts", "./deb/systemd:services", "./common/systemd:generators", + "./common:conf_files", "./common:dirs", "./common:symlinks", ":st2_venv@parametrize=py38", @@ -98,6 +99,7 @@ nfpm_rpm_package( "./rpm/scripts", "./rpm/systemd:services", "./common/systemd:generators", + "./common:conf_files", "./common:dirs", "./common:symlinks", ":st2_venv@parametrize=py38", diff --git a/packaging/common/BUILD b/packaging/common/BUILD index a9b3c0aef7..9df52d32a3 100644 --- a/packaging/common/BUILD +++ b/packaging/common/BUILD @@ -74,3 +74,16 @@ nfpm_content_symlinks( file_owner="root", file_group="root", ) + +target( + name="conf_files", + dependencies=[ + "//conf:packaged_conf_files", + "//st2actions/conf:packaged_conf_files", + "//st2api/conf:packaged_conf_files", + "//st2auth/conf:packaged_conf_files", + "//st2client/conf:packaged_bash_completion", + "//st2reactor/conf:packaged_conf_files", + "//st2stream/conf:packaged_conf_files", + ], +) diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index 9d1bf2cc2b..2b045000e7 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -21,3 +21,24 @@ st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.actionrunner.conf"), + ("syslog.conf", "/etc/st2/syslog.actionrunner.conf"), + ("logging.notifier.conf", "/etc/st2/logging.notifier.conf"), + ("syslog.notifier.conf", "/etc/st2/syslog.notifier.conf"), + ("logging.scheduler.conf", "/etc/st2/logging.scheduler.conf"), + ("syslog.scheduler.conf", "/etc/st2/syslog.scheduler.conf"), + ("logging.workflowengine.conf", "/etc/st2/logging.workflowengine.conf"), + ("syslog.workflowengine.conf", "/etc/st2/syslog.workflowengine.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index 5fc903fd3b..44e224eb9f 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -19,3 +19,20 @@ st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_gunicorn", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.api.conf"), + ("logging.gunicorn.conf", "/etc/st2/logging.api.gunicorn.conf"), + ("syslog.conf", "/etc/st2/syslog.api.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index 3300d41753..30db5d4119 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -29,3 +29,20 @@ st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_gunicorn", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.auth.conf"), + ("logging.gunicorn.conf", "/etc/st2/logging.auth.gunicorn.conf"), + ("syslog.conf", "/etc/st2/syslog.auth.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2client/conf/BUILD b/st2client/conf/BUILD index 6c95f66377..2a0cfa2543 100644 --- a/st2client/conf/BUILD +++ b/st2client/conf/BUILD @@ -1 +1,11 @@ shell_sources() + +nfpm_content_file( + name="packaged_bash_completion", + dependencies=["./st2.complete.sh"], + src="st2.complete.sh", + dst="/etc/bash_completion/st2", + file_owner="root", + file_group="root", + file_mode="rwxr-xr-x", +) diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index 4f07917387..d87e14c02b 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -13,3 +13,24 @@ st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_syslog", + ], + files=[ + ("logging.garbagecollector.conf", "/etc/st2/logging.garbagecollector.conf"), + ("syslog.garbagecollector.conf", "/etc/st2/syslog.garbagecollector.conf"), + ("logging.rulesengine.conf", "/etc/st2/logging.rulesengine.conf"), + ("syslog.rulesengine.conf", "/etc/st2/syslog.rulesengine.conf"), + ("logging.sensorcontainer.conf", "/etc/st2/logging.sensorcontainer.conf"), + ("syslog.sensorcontainer.conf", "/etc/st2/syslog.sensorcontainer.conf"), + ("logging.timersengine.conf", "/etc/st2/logging.timersengine.conf"), + ("syslog.timersengine.conf", "/etc/st2/syslog.timersengine.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index 5fc903fd3b..785a1d0604 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -19,3 +19,20 @@ st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_gunicorn", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.stream.conf"), + ("logging.gunicorn.conf", "/etc/st2/logging.stream.gunicorn.conf"), + ("syslog.conf", "/etc/st2/syslog.stream.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) From 74901f0276039b7d8085b35750fe68a5329b2445 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 22:17:34 -0600 Subject: [PATCH 23/60] Revert "Remove sysv" (st2actions/bin/runners.sh) This partially reverts commit 0f58e6a9032504a0aec22b3f1b179814e2cb25bd. This should make it easier to apply changes from st2-packages repo. --- st2actions/bin/runners.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index 916c05d820..4c5ec3692a 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -12,7 +12,15 @@ if [ -z "$WORKERS" ]; then WORKERS="${WORKERS:-4}" fi -# 1. Choose init type +# 1. Choose init type on Debian containers use sysv +if [ -x "$LSB_RELEASE" ]; then + if [ -f /.dockerenv ] && [ $($LSB_RELEASE -is) = Debian ]; then + sv=sysv + svbin=/etc/init.d/$WORKERSVC + fi +fi + +# 2. Second criteria if [ -z "$sv" -a -x $SYSTEMDCTL ]; then sv=systemd svbin=$SYSTEMDCTL @@ -20,8 +28,14 @@ elif [ -z "$sv" ] && ( /sbin/start 2>&1 | grep -q "missing job name" ); then sv=upstart svbin=$UPSTARTCTL else - >&2 echo "Unknown platform, we support ONLY upstart and systemd!" - exit 99 + # Old debians, amazon etc + sv=sysv + svbin=/etc/init.d/$WORKERSVC + if [ ! -x $svbin ]; then + >&2 echo "Init file not found: $svbin" + >&2 echo "Unknown platform, we support ONLY debian, systemd and sysv!" + exit 99 + fi fi # 2. Spwan workers @@ -33,6 +47,8 @@ while [ $i -le $WORKERS ]; do $svbin $action $SPAWNSVC@$i elif [ $sv = upstart ]; then $svbin $action $WORKERSVC WORKERID=$i + elif [ $sv = sysv ]; then + WORKERID=$i $svbin $action fi cmdrs=$? [ $cmdrs -gt 0 ] && rs=$cmdrs From 875c987c2226be3a74b66356cef89d520f491ed2 Mon Sep 17 00:00:00 2001 From: Denis Baryshev Date: Sat, 26 Dec 2015 19:59:43 +0700 Subject: [PATCH 24/60] improved init detection mechanism in runners.sh wrapper --- st2actions/bin/runners.sh | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index 4c5ec3692a..5985fbc0a6 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -1,6 +1,6 @@ #!/bin/sh -LSB_RELEASE=$(which lsb_release) +INITCOMM=$(cat /proc/1/comm) SYSTEMDCTL=/bin/systemctl UPSTARTCTL=/sbin/initctl SPAWNSVC=st2actionrunner @@ -12,23 +12,19 @@ if [ -z "$WORKERS" ]; then WORKERS="${WORKERS:-4}" fi -# 1. Choose init type on Debian containers use sysv -if [ -x "$LSB_RELEASE" ]; then - if [ -f /.dockerenv ] && [ $($LSB_RELEASE -is) = Debian ]; then - sv=sysv - svbin=/etc/init.d/$WORKERSVC - fi -fi - -# 2. Second criteria -if [ -z "$sv" -a -x $SYSTEMDCTL ]; then +## Use running init system detection criterias +# +if [ -d /run/systemd/system ]; then + # systemd is running sv=systemd svbin=$SYSTEMDCTL -elif [ -z "$sv" ] && ( /sbin/start 2>&1 | grep -q "missing job name" ); then +elif [ "$INITCOMM" = init ] && ($UPSTARTCTL version 2>&1); then + # init is running and upstart has been detected sv=upstart svbin=$UPSTARTCTL else - # Old debians, amazon etc + # In all other cases which may apply to older debians, redhats and + # centos, amazon etc. sv=sysv svbin=/etc/init.d/$WORKERSVC if [ ! -x $svbin ]; then @@ -38,7 +34,8 @@ else fi fi -# 2. Spwan workers +## Spwan workers +# action="$1"; shift; rs=0 i=1 From 17d890ccb7306f23532d79b94346c1ab0101424b Mon Sep 17 00:00:00 2001 From: Denis Baryshev Date: Fri, 22 Jan 2016 01:34:36 +0700 Subject: [PATCH 25/60] fixing upstart false positive on centos 6.6 (since we don't ship upstart jobs for rhels) --- st2actions/bin/runners.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index 5985fbc0a6..ceeaa0136b 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -18,7 +18,8 @@ if [ -d /run/systemd/system ]; then # systemd is running sv=systemd svbin=$SYSTEMDCTL -elif [ "$INITCOMM" = init ] && ($UPSTARTCTL version 2>&1); then +elif [ "$INITCOMM" = init ] && ($UPSTARTCTL version 2>&1) && + [ -f /etc/init/$WORKERSVC.conf ]; then # init is running and upstart has been detected sv=upstart svbin=$UPSTARTCTL From d4e19745818647da675dde7d542442915d7cb891 Mon Sep 17 00:00:00 2001 From: Denis Baryshev Date: Thu, 28 Jan 2016 02:33:16 +0700 Subject: [PATCH 26/60] default number of actionrunner workers is set to 10 --- st2actions/bin/runners.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index ceeaa0136b..bea4e98b43 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -5,12 +5,8 @@ SYSTEMDCTL=/bin/systemctl UPSTARTCTL=/sbin/initctl SPAWNSVC=st2actionrunner WORKERSVC=st2actionrunner-worker - # Set default number of workers -if [ -z "$WORKERS" ]; then - WORKERS=$(/usr/bin/nproc 2>/dev/null) - WORKERS="${WORKERS:-4}" -fi +WORKERS="${WORKERS:-10}" ## Use running init system detection criterias # From 603f2cec0f42b5d64c1db3c56836f24d49a2e406 Mon Sep 17 00:00:00 2001 From: Denis Baryshev Date: Fri, 4 Mar 2016 23:08:25 +0700 Subject: [PATCH 27/60] beautify runners.sh --- st2actions/bin/runners.sh | 95 ++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index bea4e98b43..1a186bec4e 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -1,52 +1,57 @@ #!/bin/sh -INITCOMM=$(cat /proc/1/comm) -SYSTEMDCTL=/bin/systemctl -UPSTARTCTL=/sbin/initctl -SPAWNSVC=st2actionrunner -WORKERSVC=st2actionrunner-worker -# Set default number of workers +# Default number of workers WORKERS="${WORKERS:-10}" -## Use running init system detection criterias -# -if [ -d /run/systemd/system ]; then - # systemd is running - sv=systemd - svbin=$SYSTEMDCTL -elif [ "$INITCOMM" = init ] && ($UPSTARTCTL version 2>&1) && - [ -f /etc/init/$WORKERSVC.conf ]; then - # init is running and upstart has been detected - sv=upstart - svbin=$UPSTARTCTL -else - # In all other cases which may apply to older debians, redhats and - # centos, amazon etc. - sv=sysv - svbin=/etc/init.d/$WORKERSVC - if [ ! -x $svbin ]; then - >&2 echo "Init file not found: $svbin" - >&2 echo "Unknown platform, we support ONLY debian, systemd and sysv!" - exit 99 +# Choose init system to perform actions with a service. +choose_sysinit() { + local service="$1" svinit="sysv" + if [ -d /run/systemd/system ]; then + svinit=systemd + elif [ "$(cat /proc/1/comm)" = init ] && [ -f /etc/init/${service}.conf ] && + (/sbin/initctl version 2>&1 | grep -q upstart); then + svinit=upstart + else + if [ ! -x /etc/init.d/${service} ]; then + >&2 echo "Supported init systems: systemd, upstart and sysv" + >&2 echo "/etc/init.d/${service} not found or disabled" + exit 99 + fi fi -fi + echo $svinit +} -## Spwan workers -# -action="$1"; shift; -rs=0 -i=1 -while [ $i -le $WORKERS ]; do - if [ $sv = systemd ]; then - $svbin $action $SPAWNSVC@$i - elif [ $sv = upstart ]; then - $svbin $action $WORKERSVC WORKERID=$i - elif [ $sv = sysv ]; then - WORKERID=$i $svbin $action - fi - cmdrs=$? - [ $cmdrs -gt 0 ] && rs=$cmdrs - i=`expr $i + 1` -done +# Perform service action over the given number of workers. +spawn_workers() { + local action=$1 + local init= seq=$(eval printf '%g\\n' {1..$WORKERS}) + + # Choose init system and exit if it's not supported. + init=$(choose_sysinit st2actionrunner) + [ $? -gt 0 ] && exit $? + + case $init in + systemd) + echo "$seq" | xargs -I{} /bin/systemctl $action \ + st2actionrunner@{} + ;; + upstart) + echo "$seq" | xargs -I{} /sbin/initctl $action \ + st2actionrunner-worker WORKERID={} + ;; + sysv) + echo "$seq" | xargs -I{} /bin/sh -c \ + "WORKERID={} /etc/init.d/st2actionrunner-worker $action" + ;; + esac + # return 1 in case if xargs failed any invoked commands. + [ $? -ge 123 ] && return 1 || return $? +} + +# Perform service action on all actionrunners +if [ -z "$1" ]; then + echo >&2 "Usage: $0 action" + exit 99 +fi -exit $rs +spawn_workers $1 From d9610877134befcdca73d605000affdd608dfaa0 Mon Sep 17 00:00:00 2001 From: Denis Baryshev Date: Mon, 7 Mar 2016 19:17:55 +0700 Subject: [PATCH 28/60] runners.sh fix xargs return code --- st2actions/bin/runners.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index 1a186bec4e..8445935376 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -23,8 +23,8 @@ choose_sysinit() { # Perform service action over the given number of workers. spawn_workers() { - local action=$1 - local init= seq=$(eval printf '%g\\n' {1..$WORKERS}) + local action=$1 init= seq= + seq=$(bash -c "printf '%g\\n' {1..$WORKERS}") # Choose init system and exit if it's not supported. init=$(choose_sysinit st2actionrunner) @@ -45,7 +45,7 @@ spawn_workers() { ;; esac # return 1 in case if xargs failed any invoked commands. - [ $? -ge 123 ] && return 1 || return $? + retval=$?; [ $retval -ge 123 ] && return 1 || return $retval } # Perform service action on all actionrunners From e212c062623e7246ac1a0c38259989fdb8cd5ce3 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 18 Jun 2020 00:07:20 +0100 Subject: [PATCH 29/60] Remove more upstart logic artifacts --- st2actions/bin/runners.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index 8445935376..17a12cf849 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -8,12 +8,9 @@ choose_sysinit() { local service="$1" svinit="sysv" if [ -d /run/systemd/system ]; then svinit=systemd - elif [ "$(cat /proc/1/comm)" = init ] && [ -f /etc/init/${service}.conf ] && - (/sbin/initctl version 2>&1 | grep -q upstart); then - svinit=upstart else if [ ! -x /etc/init.d/${service} ]; then - >&2 echo "Supported init systems: systemd, upstart and sysv" + >&2 echo "Supported init systems: systemd and sysv" >&2 echo "/etc/init.d/${service} not found or disabled" exit 99 fi @@ -35,10 +32,6 @@ spawn_workers() { echo "$seq" | xargs -I{} /bin/systemctl $action \ st2actionrunner@{} ;; - upstart) - echo "$seq" | xargs -I{} /sbin/initctl $action \ - st2actionrunner-worker WORKERID={} - ;; sysv) echo "$seq" | xargs -I{} /bin/sh -c \ "WORKERID={} /etc/init.d/st2actionrunner-worker $action" From 248a872154c6f67bf7acb7e527e4964adf502074 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 23:16:24 -0600 Subject: [PATCH 30/60] Reapply "Remove sysv" (st2actions/bin/runners.sh) Originally removed in 0f58e6a9032504a0aec22b3f1b179814e2cb25bd Reverted in 9bd37eeeffc05340453d06ecbe64b7c63f393701 Now reapplying after cherry-picking changes from st2-packages repo. --- st2actions/bin/runners.sh | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/st2actions/bin/runners.sh b/st2actions/bin/runners.sh index 17a12cf849..730d2342c7 100755 --- a/st2actions/bin/runners.sh +++ b/st2actions/bin/runners.sh @@ -5,15 +5,12 @@ WORKERS="${WORKERS:-10}" # Choose init system to perform actions with a service. choose_sysinit() { - local service="$1" svinit="sysv" + local service="$1" svinit="unknown" if [ -d /run/systemd/system ]; then svinit=systemd else - if [ ! -x /etc/init.d/${service} ]; then - >&2 echo "Supported init systems: systemd and sysv" - >&2 echo "/etc/init.d/${service} not found or disabled" - exit 99 - fi + >&2 echo "Supported init systems: ONLY systemd" + exit 99 fi echo $svinit } @@ -32,10 +29,6 @@ spawn_workers() { echo "$seq" | xargs -I{} /bin/systemctl $action \ st2actionrunner@{} ;; - sysv) - echo "$seq" | xargs -I{} /bin/sh -c \ - "WORKERID={} /etc/init.d/st2actionrunner-worker $action" - ;; esac # return 1 in case if xargs failed any invoked commands. retval=$?; [ $retval -ge 123 ] && return 1 || return $retval From 9b1eaafb728dc443ef5d1871c5165370a5ce2866 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 23:24:21 -0600 Subject: [PATCH 31/60] Packaging: Add BUILD comment about st2actions/bin/runners.sh --- st2actions/bin/BUILD | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/st2actions/bin/BUILD b/st2actions/bin/BUILD index 5a138aad51..bd2f6af922 100644 --- a/st2actions/bin/BUILD +++ b/st2actions/bin/BUILD @@ -10,3 +10,15 @@ st2_shell_sources_and_resources( sources=["*.sh"], skip_shellcheck=True, ) + +# The st2-packages repo installed this separately to avoid relying on setup.py. +# With the move to pants+pex managed virtualenv, the setup.py issue is (probably) not an issue. +# If that assumption proves wrong, use something like this to install it: +# nfpm_content_file( +# name="runners_sh", +# src="runners.sh", +# dst="/opt/stackstorm/st2/bin/runners.sh" +# file_owner="root", +# file_group="root", +# file_mode="rwxr-xr-x", +# ) From 328751bc53b1478762f94b09afd32a43631c037d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 12:59:48 -0600 Subject: [PATCH 32/60] Packaging: Add /etc/st2/htpasswd --- conf/BUILD | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index 0bc4015eee..4119e52dbb 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -58,17 +58,32 @@ file( source="st2.package.conf", ) +shell_command( + name="htpasswd", + description="An empty htpasswd file for st2-auth-backend-flat-file", + command="touch htpasswd", + tools=["touch"], + output_files=["htpasswd"] +) + nfpm_content_files( name="packaged_conf_files", dependencies=[ ":st2_package_conf", ":logrotate", + ":htpasswd", ], files=[ ("st2.package.conf", "/etc/st2/st2.conf"), ("logrotate.conf", "/etc/logrotate.d/st2"), + ("htpasswd", "/etc/st2/htpasswd"), ], file_owner="root", file_group="root", file_mode="rw-r--r--", + overrides={ + "/etc/st2/htpasswd": dict( + file_mode="rw-r-----" + ), + }, ) From 2da0f1f0a694c4608bb368c8c94a10b95730ce88 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 13:05:03 -0600 Subject: [PATCH 33/60] Packaging: Add /usr/share/doc/st2/nginx/st2.conf --- conf/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index 4119e52dbb..7fabed5d61 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -71,11 +71,13 @@ nfpm_content_files( dependencies=[ ":st2_package_conf", ":logrotate", + ":nginx_sample_config", ":htpasswd", ], files=[ ("st2.package.conf", "/etc/st2/st2.conf"), ("logrotate.conf", "/etc/logrotate.d/st2"), + ("nginx/st2.conf", "/usr/share/doc/st2/conf/nginx/st2.conf"), ("htpasswd", "/etc/st2/htpasswd"), ], file_owner="root", From e5e36881694c267a4abf77a1ea24627a28983ddd Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 14:36:18 -0600 Subject: [PATCH 34/60] Packaging: Add pre-packaging conf manipulation This was in the st2-packages.git Makefile's post_install target. --- pants-plugins/macros.py | 26 ++++++++++++++++++++++++++ st2actions/conf/BUILD | 6 +----- st2api/conf/BUILD | 6 +----- st2auth/conf/BUILD | 6 +----- st2reactor/conf/BUILD | 6 +----- st2stream/conf/BUILD | 6 +----- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 0381f20d6f..cbdb7868c0 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -227,3 +227,29 @@ def st2_logging_conf_resources(**kwargs): deps = list(deps) + list(_st2common_logging_deps) kwargs["dependencies"] = tuple(deps) resources(**kwargs) # noqa: F821 + + +def st2_logging_conf_for_nfpm(**kwargs): + deps = kwargs.pop("dependencies") or [] + + shell_command( + name="package_logging_conf", + execution_dependencies=deps, + # TODO: this will fail if using MacOS. + command="""sed -i -r "/args\\s*=\\s*/s%logs%/var/log/st2%g" logging.*conf; + sed -i "/\\[logger_root\\]/,/\\[.*\\]\\|\\s*$$/ {s/level=DEBUG/level=INFO/}" logging.*conf; + sed -i "/\\[logger_root\\]/,/\\[.*\\]\\|\\s*$$/ {s/level=DEBUG/level=INFO/}" syslog.*conf; + """, + tools=["sed"], + output_files=["*.conf"], + ) + + nfpm_content_files( + name="packaged_conf_files", + dependencies=[":package_logging_conf"], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", + content_type="config|noreplace" + **kwargs, + ) diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index 2b045000e7..fe4b057a0a 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -22,8 +22,7 @@ st2_logging_conf_files( sources=["syslog*.conf"], ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_syslog", @@ -38,7 +37,4 @@ nfpm_content_files( ("logging.workflowengine.conf", "/etc/st2/logging.workflowengine.conf"), ("syslog.workflowengine.conf", "/etc/st2/syslog.workflowengine.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index 44e224eb9f..cffb642996 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -20,8 +20,7 @@ st2_logging_conf_file( source="syslog.conf", ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_gunicorn", @@ -32,7 +31,4 @@ nfpm_content_files( ("logging.gunicorn.conf", "/etc/st2/logging.api.gunicorn.conf"), ("syslog.conf", "/etc/st2/syslog.api.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index 30db5d4119..d3a60ccbe8 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -30,8 +30,7 @@ st2_logging_conf_file( source="syslog.conf", ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_gunicorn", @@ -42,7 +41,4 @@ nfpm_content_files( ("logging.gunicorn.conf", "/etc/st2/logging.auth.gunicorn.conf"), ("syslog.conf", "/etc/st2/syslog.auth.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index d87e14c02b..593c3a6cf8 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -14,8 +14,7 @@ st2_logging_conf_files( sources=["syslog*.conf"], ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_syslog", @@ -30,7 +29,4 @@ nfpm_content_files( ("logging.timersengine.conf", "/etc/st2/logging.timersengine.conf"), ("syslog.timersengine.conf", "/etc/st2/syslog.timersengine.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index 785a1d0604..a78c5f5039 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -20,8 +20,7 @@ st2_logging_conf_file( source="syslog.conf", ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_gunicorn", @@ -32,7 +31,4 @@ nfpm_content_files( ("logging.gunicorn.conf", "/etc/st2/logging.stream.gunicorn.conf"), ("syslog.conf", "/etc/st2/syslog.stream.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) From 4e6f8a4b8bf61e3005ac670f352e31a5d8c95252 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 14:37:22 -0600 Subject: [PATCH 35/60] Packaging: update content_type for various nfpm_content_files targets --- conf/BUILD | 10 +++++++++- st2client/conf/BUILD | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/conf/BUILD b/conf/BUILD index 7fabed5d61..d497d5b7be 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -80,12 +80,20 @@ nfpm_content_files( ("nginx/st2.conf", "/usr/share/doc/st2/conf/nginx/st2.conf"), ("htpasswd", "/etc/st2/htpasswd"), ], + content_type="config|noreplace", file_owner="root", file_group="root", file_mode="rw-r--r--", overrides={ "/etc/st2/htpasswd": dict( - file_mode="rw-r-----" + # file_owner=_svc_user, + # file_group=_svc_user, + file_mode="rw-------" + ), + "/usr/share/doc/st2/conf/nginx/st2.conf": dict( + content_type="", + # file_owner=_svc_user, + # file_group=_svc_user, ), }, ) diff --git a/st2client/conf/BUILD b/st2client/conf/BUILD index 2a0cfa2543..77e1013593 100644 --- a/st2client/conf/BUILD +++ b/st2client/conf/BUILD @@ -5,6 +5,7 @@ nfpm_content_file( dependencies=["./st2.complete.sh"], src="st2.complete.sh", dst="/etc/bash_completion/st2", + content_type="config", file_owner="root", file_group="root", file_mode="rwxr-xr-x", From 96aca66b5f878fa729e13d6f2148ca2eb90c02e7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 15:01:06 -0600 Subject: [PATCH 36/60] Packaging: stub command to modify st2.package.conf --- conf/BUILD | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/BUILD b/conf/BUILD index d497d5b7be..2700dbfb26 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -66,10 +66,20 @@ shell_command( output_files=["htpasswd"] ) +shell_command( + name="packaged_st2_conf", + execution_dependencies=[":st2_package_conf"], + # TODO: how to add the if statement + # if REDHAT && [[ `$(PYTHON_ALT_BINARY) /root/scripts/platform_major_version.py` = 8 ]] + command="""sed -i -r 's/virtualenv_opts\\s*=.*/virtualenv_opts =/' st2.package.conf""", + tools=["sed"], + output_files=["st2.package.conf"] +) + nfpm_content_files( name="packaged_conf_files", dependencies=[ - ":st2_package_conf", + ":packaged_st2_conf", ":logrotate", ":nginx_sample_config", ":htpasswd", From 580afec226c5bbd07c959821326358838d455ee4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 15:28:34 -0600 Subject: [PATCH 37/60] Packaging: add TODO to conf/BUILD --- conf/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index 2700dbfb26..6760c066c7 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -95,6 +95,10 @@ nfpm_content_files( file_group="root", file_mode="rw-r--r--", overrides={ + # TODO: should st2.conf really be readable for all users? + # "/etc/st2/st2.conf": dict( + # file_mode="rw-r-----" + # ), "/etc/st2/htpasswd": dict( # file_owner=_svc_user, # file_group=_svc_user, From 928df2fc1911c12794a2c32a180823d307df2f79 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 16:46:58 -0600 Subject: [PATCH 38/60] Packaging+pants: Add BUILD macros: ST2_PACKS_GROUP and ST2_SVC_USER There is some legacy bits in st2-packages.git that attempt to pull the packs group and system user from /etc/st2/st2.conf. But, that code is not in use. Effectively, the user/group names have been hard-coded. Rather than preserve an unused install feature, the pants+nfpm based system package build will use hard-coded group/user names. --- conf/BUILD | 6 ++---- packaging/common/BUILD | 11 ++++------- pants-plugins/macros.py | 8 ++++++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/conf/BUILD b/conf/BUILD index 6760c066c7..9097d40b72 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -100,14 +100,12 @@ nfpm_content_files( # file_mode="rw-r-----" # ), "/etc/st2/htpasswd": dict( - # file_owner=_svc_user, - # file_group=_svc_user, + file_owner=ST2_SVC_USER, + file_group=ST2_SVC_USER, file_mode="rw-------" ), "/usr/share/doc/st2/conf/nginx/st2.conf": dict( content_type="", - # file_owner=_svc_user, - # file_group=_svc_user, ), }, ) diff --git a/packaging/common/BUILD b/packaging/common/BUILD index 9df52d32a3..e1a18bfb1e 100644 --- a/packaging/common/BUILD +++ b/packaging/common/BUILD @@ -1,6 +1,3 @@ -_packs_group = "st2packs" -_st2_user = "st2" - nfpm_content_dirs( name="root_dirs", dirs=[ @@ -10,8 +7,6 @@ nfpm_content_dirs( "/opt/stackstorm", "/opt/stackstorm/st2", "/usr/share/doc/st2", - "/var/log/st2", - "/var/run/st2", ], file_owner="root", file_group="root", @@ -26,7 +21,7 @@ nfpm_content_dirs( "/usr/share/doc/st2/examples", ], file_owner="root", - file_group=_packs_group, + file_group=ST2_PACKS_GROUP, file_mode="rwxrwxr-x", ) @@ -36,8 +31,10 @@ nfpm_content_dirs( "/opt/stackstorm/configs", "/opt/stackstorm/exports", "/opt/stackstorm/overrides", + "/var/log/st2", + "/var/run/st2", ], - file_owner=_st2_user, + file_owner=ST2_SVC_USER, file_group="root", file_mode="rwxr-xr-x", ) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index cbdb7868c0..0f5a631560 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -116,6 +116,10 @@ def st2_component_python_distribution(**kwargs): # https://github.com/pex-tool/pex/blob/v2.1.137/pex/common.py#L39-L45 MTIME = "1980-01-01T00:00:00Z" +# These are used for system packages (rpm/deb) +ST2_PACKS_GROUP = "st2packs" +ST2_SVC_USER = "st2" + def st2_pack_archive(**kwargs): """Create a makeself_archive using files from the given dependencies. @@ -158,7 +162,7 @@ def st2_pack_archive(**kwargs): f"/opt/stackstorm/packs/{pack_name}", # reproducibility flags: "--tar-extra", # extra tar args: '--arg=value' (equals delimited) space separated - f"--owner=root:0 --group=root:0 --mtime={MTIME} --exclude=LICENSE", # TODO: include LICENSE file? + f"--owner=root --group={ST2_PACKS_GROUP} --mtime={MTIME} --exclude=LICENSE", # TODO: include LICENSE file? "--packaging-date", MTIME, # TODO: maybe use release date instead of an epoch date? # compression/encryption flags: @@ -178,7 +182,7 @@ def st2_pack_archive(**kwargs): src=f"packaging/packs/{pack_name}.tgz.run", dst=f"/opt/stackstorm/install/packs/{pack_name}.tgz.run", file_owner="root", - file_group="root", + file_group=ST2_PACKS_GROUP, file_mode="rwxr-x---", ) From eefa69c8d7b61cfa530104acd578f4c75473f3f3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Jan 2025 13:36:16 -0600 Subject: [PATCH 39/60] Packaging: deb/pre-install.sh Remove dead code --- packaging/deb/scripts/pre-install.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index daf0adc88d..e4e5597350 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -16,7 +16,6 @@ set -e PACKS_GROUP=st2packs SYS_USER=stanley ST2_USER=st2 -ST2_CONFPATH="/etc/st2/st2.conf" ST2_UPGRADESTAMP="/tmp/.stamp-stackstorm-st2-deb-package" ## Permissions of directories which has to be reset on upgrade @@ -48,18 +47,6 @@ create_users() { fi } -## [NOT USED!] Get current system user from the st2.conf -config_sysuser() { - # exit hooked - return 0 - local sysuser= - if [ -f $ST2_CONFPATH ]; then - sysuser=$(cat $ST2_CONFPATH | - sed -n -e '/\[system_user\]/,/\[.*\]\|\$/ { /\[.*\]/d; /user\s*=/ { s/\s*user\s*=\s*//; p } }') - fi - echo $sysuser -} - ## Update logrotate configuration enable_logrotate() { [ -f /etc/logrotate.d/st2-pkgsaved.disabled ] && From b855a28e768562904281c9df205096683fc6b917 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Jan 2025 13:49:02 -0600 Subject: [PATCH 40/60] Packaging: deb/*.sh Remove logrotate config handling /etc/logrotate.d/st2 is already recorded in conffiles, so let dpkg handle conf preservation and restoration. Besides, there was a bug in the code anyway rendering this logic useless. post-remove.sh should have had mv /etc/logrotate.d/st2 /etc/logrotate.d/st2-pkgsaved.disabled but was missing the first argument, so it did nothing. This was hidden by the '|| true' that made it always pass. Since the logic doesn't work, and it is not necessary, just drop it. --- packaging/deb/scripts/post-remove.sh | 14 +++++--------- packaging/deb/scripts/pre-install.sh | 8 -------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/packaging/deb/scripts/post-remove.sh b/packaging/deb/scripts/post-remove.sh index 8469d235d7..37cd663950 100644 --- a/packaging/deb/scripts/post-remove.sh +++ b/packaging/deb/scripts/post-remove.sh @@ -32,11 +32,8 @@ st2timersengine st2workflowengine " -## Save st2 logrotate config on remove, but wipe it out on purge. -preserve_logrotate() { - if [ "$1" = remove ]; then - [ -f /etc/logrotate.d/st2 ] && mv /etc/logrotate.d/st2-pkgsaved.disabled 1>/dev/null 2>&1 || : - elif [ "$1" = purge ]; then +purge_files() { + # This -pkgsaved.disabled file might be left over from old (buggy) deb packages rm -f /etc/logrotate.d/st2-pkgsaved.disabled 1>/dev/null 2>&1 || : # Clean up other StackStorm related configs and directories rm -rf /etc/st2 1>/dev/null 2>&1 || : @@ -44,14 +41,13 @@ preserve_logrotate() { rm -rf /root/.st2 1>/dev/null 2>&1 || : rm -rf /var/log/st2 1>/dev/null 2>&1 || : rm -f /etc/sudoers.d/st2 1>/dev/null 2>&1 || : - fi } case "$1" in - remove|purge) - preserve_logrotate "$1" + purge) + purge_files ;; - upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) ;; *) echo "postrm called with unknown argument \`$1'" >&2 diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index e4e5597350..9cb0927b37 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -47,12 +47,6 @@ create_users() { fi } -## Update logrotate configuration -enable_logrotate() { - [ -f /etc/logrotate.d/st2-pkgsaved.disabled ] && - mv -f /etc/logrotate.d/st2-pkgsaved.disabled /etc/logrotate.d/st2 || : -} - ## Fix directories permissions on upgrade (different across maint scripts!) # NB! USED FOR COMPATIBILITY ON UPGRADE FROM PREVIOUS VERSIONS OF PACKAGES. # NB! In future package releases reseting permissions SHOULD BE REMOVED. @@ -77,11 +71,9 @@ set_permissions() { case "$1" in install) create_users - enable_logrotate ;; upgrade) create_users - enable_logrotate set_permissions "$RESET_PERMS" touch $ST2_UPGRADESTAMP ;; From 8b411233110642b9989edf35cdd930806c86b52c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Jan 2025 13:57:48 -0600 Subject: [PATCH 41/60] Packaging: */pre-install.sh harmonize RESET_PERMS for rpm/deb --- packaging/deb/scripts/pre-install.sh | 6 +++--- packaging/rpm/scripts/pre-install.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index 9cb0927b37..54857476a0 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -23,9 +23,9 @@ RESET_PERMS=$(cat < Date: Tue, 7 Jan 2025 14:08:48 -0600 Subject: [PATCH 42/60] Packaging: */pre-install.sh clean up RESET_PERMS Only executable files (and directories) should have the execute bit set. This update will make configs, exports, and overrides preserve the execute bit, only replacing the rw bits. --- packaging/deb/scripts/pre-install.sh | 12 ++++++------ packaging/rpm/scripts/pre-install.sh | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index 54857476a0..b84764ed6e 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -20,12 +20,12 @@ ST2_UPGRADESTAMP="/tmp/.stamp-stackstorm-st2-deb-package" ## Permissions of directories which has to be reset on upgrade RESET_PERMS=$(cat < Date: Tue, 7 Jan 2025 14:22:44 -0600 Subject: [PATCH 43/60] Packaging: Remove set_permissions functions A comment in the pre-install.sh files said: > Fix directories permissions on upgrade (different across maint scripts!) > NB! USED FOR COMPATIBILITY ON UPGRADE FROM PREVIOUS VERSIONS OF PACKAGES. > NB! In future package releases reseting permissions SHOULD BE REMOVED. This has been there for many releases. So, let's just remove this. Instead, we rely on the permissions that the st2 package provides, which are defined in BUILD nfpm_content_* metadata. This has the benefit of vastly simplifying the deb install scripts as the upgrade stamp file is no longer required. So, there is no longer a dependency between pre-install and post-install scripts. --- packaging/deb/scripts/post-install.sh | 35 ------------------------- packaging/deb/scripts/pre-install.sh | 35 ------------------------- packaging/rpm/scripts/pre-install.sh | 37 --------------------------- 3 files changed, 107 deletions(-) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index c17e4f120e..8140404417 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -17,11 +17,6 @@ set -e # for details, see http://www.debian.org/doc/debian-policy/ or # the debian-policy package -ST2_USER=st2 -PACKS_GROUP=st2packs -ST2_UPGRADESTAMP="/tmp/.stamp-stackstorm-st2-deb-package" -upgrading=0 - _ST2_SERVICES=" st2actionrunner.service st2api.service @@ -36,38 +31,8 @@ st2timersengine.service st2workflowengine.service " -## Permissions of files which should be set on install -SET_PERMS=$(cat </dev/null 2>&1 || true ;; diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index b84764ed6e..5d327b21ae 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -16,18 +16,6 @@ set -e PACKS_GROUP=st2packs SYS_USER=stanley ST2_USER=st2 -ST2_UPGRADESTAMP="/tmp/.stamp-stackstorm-st2-deb-package" - -## Permissions of directories which has to be reset on upgrade -RESET_PERMS=$(cat < 1) -if [ "$1" -gt 1 ]; then - set_permissions "$RESET_PERMS" -fi From 5fe1751820744a29bbd7e08d8dc6105f0591f43d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Jan 2025 18:40:29 -0600 Subject: [PATCH 44/60] Packaging: Improve rpm scriptlet / deb maintainer script docs Document all scripts, and update deb docs to better describe when various args might be used and add some missing variants. --- packaging/deb/scripts/post-install.sh | 32 ++++++++++++++++++--------- packaging/deb/scripts/post-remove.sh | 28 ++++++++++++++--------- packaging/deb/scripts/pre-install.sh | 15 ++++++++----- packaging/deb/scripts/pre-remove.sh | 15 +++++++++++++ packaging/rpm/scripts/post-install.sh | 6 +++++ packaging/rpm/scripts/post-remove.sh | 6 +++++ packaging/rpm/scripts/pre-install.sh | 6 +++++ packaging/rpm/scripts/pre-remove.sh | 6 +++++ 8 files changed, 87 insertions(+), 27 deletions(-) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 8140404417..7570b1a8cd 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -6,16 +6,28 @@ set -e # summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package +# configure +# on fresh install +# configure +# on upgrade OR on install after pkg removal without purging conf files +# abort-upgrade +# on upgrade failed (after failure of prerm, preinst, postrm) +# abort-remove in-favour +# on removal due to conflict with other package +# abort-remove +# on removal (after failure of prerm) +# abort-deconfigure in-favour +# +# [ removing ] +# on removal due to breaks/conflict with other package (if --auto-deconfigure) +# triggered [ ...] +# when a trigger we've registered interest in fires, +# such as when /usr/bin/python3.9 (or similar) gets updated, +# allowing this script to rebuild the venv. +# https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html +# https://www.mankier.com/5/deb-postinst +# https://www.mankier.com/5/deb-triggers +# https://stackoverflow.com/questions/15276535/dpkg-how-to-use-trigger _ST2_SERVICES=" st2actionrunner.service diff --git a/packaging/deb/scripts/post-remove.sh b/packaging/deb/scripts/post-remove.sh index 37cd663950..cf6521bd58 100644 --- a/packaging/deb/scripts/post-remove.sh +++ b/packaging/deb/scripts/post-remove.sh @@ -6,17 +6,23 @@ set -e # summary of how this script can be called: -# * `remove' -# * `purge' -# * `upgrade' -# * `failed-upgrade' -# * `abort-install' -# * `abort-install' -# * `abort-upgrade' -# * `disappear' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package +# remove +# on remove or remove+purge +# purge +# on purge or remove+purge +# upgrade +# on upgrade +# disappear +# on implicit removal (all package files replaced by another package) +# abort-install +# on failed fresh install (after failed) +# abort-install +# on failed install after pkg removal w/o conf purge (and failed) +# failed-upgrade +# on upgrade failed (after failed) +# abort-upgrade +# on upgrade failed (after or failed) +# https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html _ST2_SERVICES=" st2actionrunner diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index 5d327b21ae..9e5f935c8e 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -6,12 +6,15 @@ set -e # summary of how this script can be called: -# * `install' -# * `install' -# * `upgrade' -# * `abort-upgrade' -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package +# install +# on fresh install +# install +# on install after pkg removal without conf purge +# upgrade +# on upgrade +# abort-upgrade +# on upgrade failed (after failure of postrm) +# https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html PACKS_GROUP=st2packs SYS_USER=stanley diff --git a/packaging/deb/scripts/pre-remove.sh b/packaging/deb/scripts/pre-remove.sh index be1af9039e..45faaa2cbd 100644 --- a/packaging/deb/scripts/pre-remove.sh +++ b/packaging/deb/scripts/pre-remove.sh @@ -3,6 +3,21 @@ set -e +# summary of how this script can be called: +# remove +# on remove or remove+purge +# upgrade +# on upgrade +# remove in-favour +# on removal due to conflict with other package +# deconfigure in-favour +# +# [ removing ] +# on removal due to breaks/conflict with other package (if --auto-deconfigure) +# failed-upgrade +# on upgrade failed (after failed) +# https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html + _ST2_SERVICES=" st2actionrunner st2api diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index 67d20480b9..baece97289 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -1,5 +1,11 @@ set -e +# This %post scriptlet gets one argument, $1, the number of packages of +# this name that will be left on the system when this script completes. So: +# * on install: $1 = 1 +# * on upgrade: $1 > 1 +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax + _ST2_SERVICES=" st2actionrunner st2api diff --git a/packaging/rpm/scripts/post-remove.sh b/packaging/rpm/scripts/post-remove.sh index 0f08be40b3..3e93508d5a 100644 --- a/packaging/rpm/scripts/post-remove.sh +++ b/packaging/rpm/scripts/post-remove.sh @@ -1,5 +1,11 @@ set -e +# This %postun scriptlet gets one argument, $1, the number of packages of +# this name that will be left on the system when this script completes. So: +# * on upgrade: $1 > 0 +# * on uninstall: $1 = 0 +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax + _ST2_SERVICES=" st2actionrunner st2actionrunner@ diff --git a/packaging/rpm/scripts/pre-install.sh b/packaging/rpm/scripts/pre-install.sh index 34dfa81abf..a13e9d2134 100644 --- a/packaging/rpm/scripts/pre-install.sh +++ b/packaging/rpm/scripts/pre-install.sh @@ -1,5 +1,11 @@ set -e +# This %pre scriptlet gets one argument, $1, the number of packages of +# this name that will be left on the system when this script completes. So: +# * on install: $1 = 1 +# * on upgrade: $1 > 1 +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax + PACKS_GROUP=st2packs SYS_USER=stanley ST2_USER=st2 diff --git a/packaging/rpm/scripts/pre-remove.sh b/packaging/rpm/scripts/pre-remove.sh index aeecd70401..0a70cb8ea0 100644 --- a/packaging/rpm/scripts/pre-remove.sh +++ b/packaging/rpm/scripts/pre-remove.sh @@ -1,5 +1,11 @@ set -e +# This %preun scriptlet gets one argument, $1, the number of packages of +# this name that will be left on the system when this script completes. So: +# * on upgrade: $1 > 0 +# * on uninstall: $1 = 0 +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax + _ST2_SERVICES=" st2actionrunner st2actionrunner@ From f40f26c2ba48782a85176826c9b86eaf08985978 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Jan 2025 19:20:58 -0600 Subject: [PATCH 45/60] Packaging: harmonize rpm/deb pre-install scripts Consistent formatting (indents), and comments explaining why create_user() passes different args to adduser (EL and debian have different adduser binaries that take different args) --- packaging/deb/scripts/pre-install.sh | 46 ++++++++++++---------------- packaging/rpm/scripts/pre-install.sh | 22 ++++++------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/packaging/deb/scripts/pre-install.sh b/packaging/deb/scripts/pre-install.sh index 9e5f935c8e..8d9ecf3bdc 100644 --- a/packaging/deb/scripts/pre-install.sh +++ b/packaging/deb/scripts/pre-install.sh @@ -20,42 +20,34 @@ PACKS_GROUP=st2packs SYS_USER=stanley ST2_USER=st2 -## Create stackstorm users and groups +## Create stackstorm users and groups (adduser differs from EL) create_users() { - # create st2 user (services user) - (id $ST2_USER 1>/dev/null 2>&1) || - adduser --group --disabled-password --no-create-home --system $ST2_USER - - # make st2 member of st2packs group - (getent group $PACKS_GROUP 1>/dev/null 2>&1) || groupadd -r $PACKS_GROUP - (groups $ST2_USER 2>/dev/null | grep -q "\b${PACKS_GROUP}\b") || - usermod -a -G $PACKS_GROUP $ST2_USER - - # create stanley user (for actionrunner service) - if (! id $SYS_USER 1>/dev/null 2>&1); then - adduser --group $SYS_USER - adduser --disabled-password --gecos "" --ingroup $SYS_USER $SYS_USER - fi + # create st2 user (services user) + (id $ST2_USER 1>/dev/null 2>&1) || + adduser --group --disabled-password --no-create-home --system $ST2_USER + + # make st2 member of st2packs group + (getent group $PACKS_GROUP 1>/dev/null 2>&1) || groupadd -r $PACKS_GROUP + (groups $ST2_USER 2>/dev/null | grep -q "\b${PACKS_GROUP}\b") || + usermod -a -G $PACKS_GROUP $ST2_USER + + # create stanley user (for actionrunner service) + if (! id $SYS_USER 1>/dev/null 2>&1); then + adduser --group $SYS_USER + adduser --disabled-password --gecos "" --ingroup $SYS_USER $SYS_USER + fi } case "$1" in - install) - create_users - ;; - upgrade) - create_users + install|upgrade) + create_users ;; abort-upgrade) ;; *) - echo "preinst called with unknown argument \`$1'" >&2 - exit 1 + # echo "preinst called with unknown argument \`$1'" >&2 + # exit 1 ;; esac -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - exit 0 diff --git a/packaging/rpm/scripts/pre-install.sh b/packaging/rpm/scripts/pre-install.sh index a13e9d2134..3a6c46a106 100644 --- a/packaging/rpm/scripts/pre-install.sh +++ b/packaging/rpm/scripts/pre-install.sh @@ -10,20 +10,20 @@ PACKS_GROUP=st2packs SYS_USER=stanley ST2_USER=st2 -## Create stackstorm users and groups (differs from debian) +## Create stackstorm users and groups (adduser differs from debian) create_users() { - # create st2 user (services user) - (id $ST2_USER 1>/dev/null 2>&1) || - adduser --no-create-home --system --user-group $ST2_USER + # create st2 user (services user) + (id $ST2_USER 1>/dev/null 2>&1) || + adduser --no-create-home --system --user-group $ST2_USER - # make st2 member of st2packs group - (getent group $PACKS_GROUP 1>/dev/null 2>&1) || groupadd -r $PACKS_GROUP - (groups $ST2_USER 2>/dev/null | grep -q "\b${PACKS_GROUP}\b") || - usermod -a -G $PACKS_GROUP $ST2_USER + # make st2 member of st2packs group + (getent group $PACKS_GROUP 1>/dev/null 2>&1) || groupadd -r $PACKS_GROUP + (groups $ST2_USER 2>/dev/null | grep -q "\b${PACKS_GROUP}\b") || + usermod -a -G $PACKS_GROUP $ST2_USER - # create stanley user (unprivileged action user, we don't ship sudoers.d config) - (id $SYS_USER 1>/dev/null 2>&1) || - adduser --user-group $SYS_USER + # create stanley user (unprivileged action user, we don't ship sudoers.d config) + (id $SYS_USER 1>/dev/null 2>&1) || + adduser --user-group $SYS_USER } create_users From 03468730c7a723920ceb5f642ec70bbc3d4c39a1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 09:48:09 -0600 Subject: [PATCH 46/60] Packaging: fmt deb scripts - Consistent formatting (indents) - do not fail scripts for unknown arguments (likely added in newer versions of dpkg) --- packaging/deb/scripts/post-install.sh | 24 ++++++------------------ packaging/deb/scripts/post-remove.sh | 13 ++----------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 7570b1a8cd..1b97b59704 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -45,15 +45,14 @@ st2workflowengine.service case "$1" in configure) - # make sure that our socket generators run - systemctl daemon-reload >/dev/null 2>&1 || true + # make sure that our socket generators run + systemctl daemon-reload >/dev/null 2>&1 || true ;; abort-upgrade|abort-remove|abort-deconfigure) ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 + # echo "postinst called with unknown argument \`$1'" >&2 + # exit 1 ;; esac @@ -76,22 +75,11 @@ if [ "$1" = "configure" ] || [ "$1" = "abort-upgrade" ] || [ "$1" = "abort-decon done systemctl --system daemon-reload >/dev/null || true if [ -n "$2" ]; then - _dh_action=restart + _dh_action=restart else - _dh_action=start + _dh_action=start fi deb-systemd-invoke $_dh_action ${_ST2_SERVICES} >/dev/null || true fi -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. -# includes: -# dh_python2: dh-virtualenv -# dh_systemd_enable/12.10ubuntu1 -# dh_systemd_start/12.10ubuntu1 - -#DEBHELPER# - - - exit 0 diff --git a/packaging/deb/scripts/post-remove.sh b/packaging/deb/scripts/post-remove.sh index cf6521bd58..2cd11476b0 100644 --- a/packaging/deb/scripts/post-remove.sh +++ b/packaging/deb/scripts/post-remove.sh @@ -56,8 +56,8 @@ case "$1" in remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) ;; *) - echo "postrm called with unknown argument \`$1'" >&2 - exit 1 + # echo "postrm called with unknown argument \`$1'" >&2 + # exit 1 ;; esac @@ -82,13 +82,4 @@ for service in ${_ST2_SERVICES}; do fi done -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. -# includes: -# dh_python2: dh-virtualenv -# dh_systemd_enable/12.10ubuntu1 -# dh_systemd_start/12.10ubuntu1 - -#DEBHELPER# - exit 0 From a871446e02ab8c339784d0b72f725391ae8f5921 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 11:11:36 -0600 Subject: [PATCH 47/60] Packaging: Improve systemd logic in deb/scripts/post-install.sh I initially copied the logic from some of the rpm's generated with native deb tooling. I wanted to link to the actual sources, however, to facilitate comparing the logic for future updates. --- packaging/deb/scripts/post-install.sh | 71 ++++++++++++++++----------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 1b97b59704..59bb1a10b6 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -29,6 +29,7 @@ set -e # https://www.mankier.com/5/deb-triggers # https://stackoverflow.com/questions/15276535/dpkg-how-to-use-trigger +# This must include ".service" to satisfy deb-systemd-{helper,invoke} _ST2_SERVICES=" st2actionrunner.service st2api.service @@ -43,12 +44,52 @@ st2timersengine.service st2workflowengine.service " +# Native .deb maintainer scripts are injected with debhelper snippets. +# We are using nfpm, so we inline those snippets here. +# https://github.com/Debian/debhelper/blob/debian/12.10/dh_systemd_start +# https://github.com/Debian/debhelper/blob/debian/12.10/dh_systemd_enable +# https://github.com/Debian/debhelper/blob/debian/12.10/autoscripts/postinst-systemd-enable +# https://github.com/Debian/debhelper/blob/debian/12.10/autoscripts/postinst-systemd-restart + +systemd_enable() { + # This will only remove masks created by d-s-h on package removal. + deb-systemd-helper unmask "${1}" >/dev/null || true + + # was-enabled defaults to true, so new installations run enable. + if deb-systemd-helper --quiet was-enabled "${1}"; then + # Enables the unit on first installation, creates new + # symlinks on upgrades if the unit file has changed. + deb-systemd-helper enable "${1}" >/dev/null || true + else + # Update the statefile to add new symlinks (if any), which need to be + # cleaned up on purge. Also remove old symlinks. + deb-systemd-helper update-state "${1}" >/dev/null || true + fi +} + +if [ -n "$2" ]; then + _dh_action=restart +else + _dh_action=start +fi + +systemd_enable_and_restart() { + for service in ${@}; do + systmd_enable "${service}" + done + if [ -d /run/systemd/system ]; then + systemctl --system daemon-reload >/dev/null || true + deb-systemd-invoke $_dh_action ${@} >/dev/null || true + fi +} + case "$1" in configure) - # make sure that our socket generators run - systemctl daemon-reload >/dev/null 2>&1 || true + systemd_enable_and_restart ${_ST2_SERVICES} ;; abort-upgrade|abort-remove|abort-deconfigure) + # dh_systemd_* runs this for all actions, not just configure + systemd_enable_and_restart ${_ST2_SERVICES} ;; *) # echo "postinst called with unknown argument \`$1'" >&2 @@ -56,30 +97,4 @@ case "$1" in ;; esac -# based on dh_systemd_enable/12.10ubuntu1 and dh_systemd_start/12.10ubuntu1 -if [ "$1" = "configure" ] || [ "$1" = "abort-upgrade" ] || [ "$1" = "abort-deconfigure" ] || [ "$1" = "abort-remove" ] ; then - for service in ${_ST2_SERVICES}; do - # This will only remove masks created by d-s-h on package removal. - deb-systemd-helper unmask "${service}" >/dev/null || true - - # was-enabled defaults to true, so new installations run enable. - if deb-systemd-helper --quiet was-enabled "${service}"; then - # Enables the unit on first installation, creates new - # symlinks on upgrades if the unit file has changed. - deb-systemd-helper enable "${service}" >/dev/null || true - else - # Update the statefile to add new symlinks (if any), which need to be - # cleaned up on purge. Also remove old symlinks. - deb-systemd-helper update-state "${service}" >/dev/null || true - fi - done - systemctl --system daemon-reload >/dev/null || true - if [ -n "$2" ]; then - _dh_action=restart - else - _dh_action=start - fi - deb-systemd-invoke $_dh_action ${_ST2_SERVICES} >/dev/null || true -fi - exit 0 From cc55f5b4d4506e1121180750a8bdca98fab2931e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 11:42:44 -0600 Subject: [PATCH 48/60] Packaging: Improve systemd logic in deb/scripts/{post,pre}-remove.sh I initially copied the logic from some of the rpm's generated with native deb tooling. I wanted to link to the actual sources, however, to facilitate comparing the logic for future updates. --- packaging/deb/scripts/post-remove.sh | 78 ++++++++++++++++------------ packaging/deb/scripts/pre-remove.sh | 51 ++++++++++++------ 2 files changed, 81 insertions(+), 48 deletions(-) diff --git a/packaging/deb/scripts/post-remove.sh b/packaging/deb/scripts/post-remove.sh index 2cd11476b0..9074f384e0 100644 --- a/packaging/deb/scripts/post-remove.sh +++ b/packaging/deb/scripts/post-remove.sh @@ -24,20 +24,47 @@ set -e # on upgrade failed (after or failed) # https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html +# This must include ".service" to satisfy deb-systemd-helper _ST2_SERVICES=" -st2actionrunner -st2api -st2auth -st2garbagecollector -st2notifier -st2rulesengine -st2scheduler -st2sensorcontainer -st2stream -st2timersengine -st2workflowengine +st2actionrunner.service +st2api.service +st2auth.service +st2garbagecollector.service +st2notifier.service +st2rulesengine.service +st2scheduler.service +st2sensorcontainer.service +st2stream.service +st2timersengine.service +st2workflowengine.service " +# Native .deb maintainer scripts are injected with debhelper snippets. +# We are using nfpm, so we inline those snippets here. +# https://github.com/Debian/debhelper/blob/debian/12.10/dh_systemd_start +# https://github.com/Debian/debhelper/blob/debian/12.10/dh_systemd_enable +# https://github.com/Debian/debhelper/blob/debian/12.10/autoscripts/postrm-systemd +# https://github.com/Debian/debhelper/blob/debian/12.10/autoscripts/postrm-systemd-reload-only + +systemd_remove() { + if [ -x "/usr/bin/deb-systemd-helper" ]; then + deb-systemd-helper mask ${@} >/dev/null || true + fi +} + +systemd_purge() { + if [ -x "/usr/bin/deb-systemd-helper" ]; then + deb-systemd-helper purge ${@} >/dev/null || true + deb-systemd-helper unmask ${@} >/dev/null || true + fi +} + +systemd_reload() { + if [ -d "/run/systemd/system" ]; then + systemctl --system daemon-reload >/dev/null || true + fi +} + purge_files() { # This -pkgsaved.disabled file might be left over from old (buggy) deb packages rm -f /etc/logrotate.d/st2-pkgsaved.disabled 1>/dev/null 2>&1 || : @@ -50,10 +77,16 @@ purge_files() { } case "$1" in + remove) + systemd_remove ${_ST2_SERVICES} + systemd_reload + ;; purge) + systemd_purge ${_ST2_SERVICES} + systemd_reload purge_files ;; - remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) ;; *) # echo "postrm called with unknown argument \`$1'" >&2 @@ -61,25 +94,4 @@ case "$1" in ;; esac -# based on dh_systemd_start/12.10ubuntu1 -if [ -d /run/systemd/system ]; then - systemctl --system daemon-reload >/dev/null || true -fi - -for service in ${_ST2_SERVICES}; do - # based on dh_systemd_enable/12.10ubuntu1 and dh_systemd_start/12.10ubuntu1 - if [ "$1" = "remove" ]; then - if [ -x "/usr/bin/deb-systemd-helper" ]; then - deb-systemd-helper mask "${service}" >/dev/null || true - fi - fi - - if [ "$1" = "purge" ]; then - if [ -x "/usr/bin/deb-systemd-helper" ]; then - deb-systemd-helper purge "${service}" >/dev/null || true - deb-systemd-helper unmask "${service}" >/dev/null || true - fi - fi -done - exit 0 diff --git a/packaging/deb/scripts/pre-remove.sh b/packaging/deb/scripts/pre-remove.sh index 45faaa2cbd..1d69e8c97a 100644 --- a/packaging/deb/scripts/pre-remove.sh +++ b/packaging/deb/scripts/pre-remove.sh @@ -18,21 +18,42 @@ set -e # on upgrade failed (after failed) # https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html +# This must include ".service" to satisfy deb-systemd-invoke _ST2_SERVICES=" -st2actionrunner -st2api -st2auth -st2garbagecollector -st2notifier -st2rulesengine -st2scheduler -st2sensorcontainer -st2stream -st2timersengine -st2workflowengine +st2actionrunner.service +st2api.service +st2auth.service +st2garbagecollector.service +st2notifier.service +st2rulesengine.service +st2scheduler.service +st2sensorcontainer.service +st2stream.service +st2timersengine.service +st2workflowengine.service " -# based on dh_systemd_start/12.10ubuntu1 -if [ -d /run/systemd/system ] && [ "$1" = remove ]; then - systemctl stop ${_ST2_SERVICES} >/dev/null || true -fi +# Native .deb maintainer scripts are injected with debhelper snippets. +# We are using nfpm, so we inline those snippets here. +# https://github.com/Debian/debhelper/blob/debian/12.10/dh_systemd_start +# https://github.com/Debian/debhelper/blob/debian/12.10/autoscripts/prerm-systemd-restart + +systemd_stop() { + if [ -d "/run/systemd/system" ]; then + deb-systemd-invoke stop ${@} >/dev/null || true + fi +} + +case "$1" in + remove) + systemd_stop ${_ST2_SERVICES} + ;; + upgrade|deconfigure|failed-upgrade) + ;; + *) + # echo "prerm called with unknown argument \`$1'" >&2 + # exit 1 + ;; +esac + +exit 0 From 48f074796064dda65f8fd9b7f9549ec69dc946aa Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 14:42:09 -0600 Subject: [PATCH 49/60] Packaging: Finish systemd logic in rpm/scripts I initially copied the logic from some of the rpm's generated with native rpm tooling. I wanted to link to the actual sources, however, to facilitate comparing the logic for future updates. st2.spec used %service_* macros which were defined in helper.spec which used %systemd_* macros. Now, I've linked to the actual systemd macro sources and cleanly combined the logic for EL8+9. --- packaging/rpm/scripts/post-install.sh | 20 ++++++++++++-------- packaging/rpm/scripts/post-remove.sh | 18 +++++++++++------- packaging/rpm/scripts/pre-remove.sh | 18 +++++++++++------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index baece97289..eae1ce969e 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -20,18 +20,22 @@ st2timersengine st2workflowengine " -# EL 8: %service_post +# Native .rpm specs use macros that get expanded into shell snippets. +# We are using nfpm, so we inline the macro expansion here. +# %systemd_post +# EL8: https://github.com/systemd/systemd/blob/v239/src/core/macros.systemd.in +# EL9: https://github.com/systemd/systemd/blob/v252/src/rpm/macros.systemd.in + if [ $1 -eq 1 ] ; then # Initial installation - systemctl --no-reload preset ${_ST2_SERVICES} &>/dev/null || : -fi -# EL 9: %service_post -if [ $1 -eq 1 ] && [ -x "/usr/lib/systemd/systemd-update-helper" ]; then - # Initial installation - /usr/lib/systemd/systemd-update-helper install-system-units ${_ST2_SERVICES} || : + if [ -x "/usr/lib/systemd/systemd-update-helper" ]; then # EL 9 + /usr/lib/systemd/systemd-update-helper install-system-units ${_ST2_SERVICES} || : + else # EL 8 + systemctl --no-reload preset ${_ST2_SERVICES} &>/dev/null || : + fi fi systemctl --no-reload enable ${_ST2_SERVICES} &>/dev/null || : -# make sure that our socket generators run +# make sure that our socket/unit generators run systemctl daemon-reload &>/dev/null || : diff --git a/packaging/rpm/scripts/post-remove.sh b/packaging/rpm/scripts/post-remove.sh index 3e93508d5a..481cfead55 100644 --- a/packaging/rpm/scripts/post-remove.sh +++ b/packaging/rpm/scripts/post-remove.sh @@ -21,15 +21,19 @@ st2timersengine st2workflowengine " -# EL 8: %service_postun +# Native .rpm specs use macros that get expanded into shell snippets. +# We are using nfpm, so we inline the macro expansion here. +# %systemd_postun_with_restart +# EL8: https://github.com/systemd/systemd/blob/v239/src/core/macros.systemd.in +# EL9: https://github.com/systemd/systemd/blob/v252/src/rpm/macros.systemd.in + if [ $1 -ge 1 ] ; then # Package upgrade, not uninstall - systemctl try-restart ${_ST2_SERVICES} &>/dev/null || : -fi -# EL 9: %service_postun -if [ $1 -ge 1 ] && [ -x "/usr/lib/systemd/systemd-update-helper" ]; then - # Package upgrade, not uninstall - /usr/lib/systemd/systemd-update-helper mark-restart-system-units ${_ST2_SERVICES} || : + if [ -x "/usr/lib/systemd/systemd-update-helper" ]; then # EL 9 + /usr/lib/systemd/systemd-update-helper mark-restart-system-units ${_ST2_SERVICES} || : + else # EL 8 + systemctl try-restart ${_ST2_SERVICES} &>/dev/null || : + fi fi # Remove st2 logrotate config, since there's no analog of apt-get purge available diff --git a/packaging/rpm/scripts/pre-remove.sh b/packaging/rpm/scripts/pre-remove.sh index 0a70cb8ea0..2df042cfff 100644 --- a/packaging/rpm/scripts/pre-remove.sh +++ b/packaging/rpm/scripts/pre-remove.sh @@ -21,13 +21,17 @@ st2timersengine st2workflowengine " -# EL 8: %service_preun +# Native .rpm specs use macros that get expanded into shell snippets. +# We are using nfpm, so we inline the macro expansion here. +# %systemd_preun +# EL8: https://github.com/systemd/systemd/blob/v239/src/core/macros.systemd.in +# EL9: https://github.com/systemd/systemd/blob/v252/src/rpm/macros.systemd.in + if [ $1 -eq 0 ] ; then # Package removal, not upgrade - systemctl --no-reload disable --now ${_ST2_SERVICES} &>/dev/null || : -fi -# EL 9: %service_preun -if [ $1 -eq 0 ] && [ -x "/usr/lib/systemd/systemd-update-helper" ]; then - # Package removal, not upgrade - /usr/lib/systemd/systemd-update-helper remove-system-units ${_ST2_SERVICES} || : + if [ -x "/usr/lib/systemd/systemd-update-helper" ]; then # EL 9 + /usr/lib/systemd/systemd-update-helper remove-system-units ${_ST2_SERVICES} || : + else # EL 8 + systemctl --no-reload disable --now ${_ST2_SERVICES} &>/dev/null || : + fi fi From c46653267c79e5da0f572fed4603d21d0c72684f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 14:56:25 -0600 Subject: [PATCH 50/60] Packaging: Use systemd-update-helper in rpm/scripts/post-install.sh --- packaging/rpm/scripts/post-install.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index eae1ce969e..c1f408ab8a 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -35,7 +35,12 @@ if [ $1 -eq 1 ] ; then fi fi +# TODO: Maybe remove this as 'preset' (on install above) enables units by default systemctl --no-reload enable ${_ST2_SERVICES} &>/dev/null || : # make sure that our socket/unit generators run -systemctl daemon-reload &>/dev/null || : +if [ -x "/usr/lib/systemd/systemd-update-helper" ]; then # EL 9 + /usr/lib/systemd/systemd-update-helper system-reload || : +else # EL 8 + systemctl daemon-reload &>/dev/null || : +fi From 1efa769b47839057ce52b231a90066c740029d54 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 15:39:07 -0600 Subject: [PATCH 51/60] Packaging: Build st2 virtualenv in post-install.sh scripts --- packaging/deb/scripts/post-install.sh | 7 +++++++ packaging/rpm/scripts/post-install.sh | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 59bb1a10b6..6128c1ca5e 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -83,8 +83,15 @@ systemd_enable_and_restart() { fi } +rebuild_st2_venv() { + /opt/stackstorm/install/st2.pex +} + case "$1" in configure) + # Fail install if venv build fails + rebuild_st2_venv || exit $? + systemd_enable_and_restart ${_ST2_SERVICES} ;; abort-upgrade|abort-remove|abort-deconfigure) diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index c1f408ab8a..d3f2eba6f6 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -20,6 +20,13 @@ st2timersengine st2workflowengine " +rebuild_st2_venv() { + /opt/stackstorm/install/st2.pex +} + +# Fail install if venv build fails +rebuild_st2_venv || exit $? + # Native .rpm specs use macros that get expanded into shell snippets. # We are using nfpm, so we inline the macro expansion here. # %systemd_post From cfd41f533c34faa5f50d46cf0383defcac2ec8c3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Jan 2025 15:40:12 -0600 Subject: [PATCH 52/60] Packaging: Extract st2 packs in post-install.sh scripts --- packaging/deb/scripts/post-install.sh | 21 ++++++++++++++++++++- packaging/rpm/scripts/post-install.sh | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 6128c1ca5e..4a9af7eb84 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -29,6 +29,15 @@ set -e # https://www.mankier.com/5/deb-triggers # https://stackoverflow.com/questions/15276535/dpkg-how-to-use-trigger +# The default set of packs installed with st2. +_ST2_PACKS=" +chatops +core +default +linux +packs +" + # This must include ".service" to satisfy deb-systemd-{helper,invoke} _ST2_SERVICES=" st2actionrunner.service @@ -87,10 +96,20 @@ rebuild_st2_venv() { /opt/stackstorm/install/st2.pex } +extract_st2_pack() { + pack=${1} + shift + PAGER=cat /opt/stackstorm/install/packs/${pack}.tgz.run --quiet --accept ${@} +} + case "$1" in configure) - # Fail install if venv build fails + # Fail install if venv build or pack extraction fails rebuild_st2_venv || exit $? + for pack in ${_ST2_PACKS}; do + extract_st2_pack ${pack} || exit $? + done + extract_st2_pack examples --target /usr/share/doc/st2/examples || : systemd_enable_and_restart ${_ST2_SERVICES} ;; diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index d3f2eba6f6..21216b3a72 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -6,6 +6,15 @@ set -e # * on upgrade: $1 > 1 # https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax +# The default set of packs installed with st2. +_ST2_PACKS=" +chatops +core +default +linux +packs +" + _ST2_SERVICES=" st2actionrunner st2api @@ -24,8 +33,18 @@ rebuild_st2_venv() { /opt/stackstorm/install/st2.pex } -# Fail install if venv build fails +extract_st2_pack() { + pack=${1} + shift + PAGER=cat /opt/stackstorm/install/packs/${pack}.tgz.run --quiet --accept ${@} +} + +# Fail install if venv build or pack extraction fails rebuild_st2_venv || exit $? +for pack in ${_ST2_PACKS}; do + extract_st2_pack ${pack} || exit $? +done +extract_st2_pack examples --target /usr/share/doc/st2/examples || : # Native .rpm specs use macros that get expanded into shell snippets. # We are using nfpm, so we inline the macro expansion here. From a881c7723c3d80c94ab8c1b46ad1a19f81cceb1a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Jan 2025 10:57:30 -0600 Subject: [PATCH 53/60] Packaging: Use .pants.bootstrap + env var to get version into BUILD files I would rather inject this from the release plugin. I'll have to work on that feature in the pants repo. --- .pants.bootstrap | 10 ++++++++++ packaging/BUILD | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .pants.bootstrap diff --git a/.pants.bootstrap b/.pants.bootstrap new file mode 100644 index 0000000000..494bd6c5b9 --- /dev/null +++ b/.pants.bootstrap @@ -0,0 +1,10 @@ +# This file should be extremely light-weight. +# Pants sources this file with something like: +# /usr/bin/env bash -c 'set -eou pipefail; source .pants.bootstrap; exec pants' +# Relevant docs: +# https://www.pantsbuild.org/blog/2024/04/27/simple-versioning-with-git-tags +# https://www.pantsbuild.org/stable/docs/using-pants/key-concepts/options#pantsbootstrap-file +# https://github.com/pantsbuild/scie-pants/blob/main/src/main.rs + +# This uses " as the cut delimiter because fmt with black uses " for strings. +export ST2PKG_VERSION="$(grep __version__ st2common/st2common/__init__.py | cut -d'"' -f2)" diff --git a/packaging/BUILD b/packaging/BUILD index 99c530bd3b..d04af10357 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -5,6 +5,10 @@ python_sources() # - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package # - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ +_st2_version = env("ST2PKG_VERSION") or "" # from .pants.bootstrap +# semver needs a dash before prerelease, so convert: 3.9dev => 3.9-dev +_st2_version = _st2_version.replace("dev", "-dev") + _pkg_description = """ StackStorm Event-driven automation Package is full standalone st2 installation including all components @@ -16,7 +20,9 @@ _common_pkg_metadata = dict( homepage="https://stackstorm.com", # https://jfearn.fedorapeople.org/en-US/RPM/4/html-single/RPM_Guide/index.html#idp3030720 license="Apache-2.0", # TODO: nFPM is putting this under Copyright tag instead of License - version="", # TODO: where does version come from? + version_schema="semver", + version=_st2_version, # parsed into version[-version_prerelease][+version_metadata] + # TODO: version_release # arch used to be "any", but that was not correct as the venv has compiled packages. arch="amd64", # TODO: parametrize this? platform="linux", From 78f6c0dd5324082c4359c03a6aa18ac0263ff54d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Jan 2025 21:14:39 -0600 Subject: [PATCH 54/60] pants-plugins/release: separate version extraction rule --- pants-plugins/release/rules.py | 55 ++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/pants-plugins/release/rules.py b/pants-plugins/release/rules.py index 86df6c0461..2bdc6931eb 100644 --- a/pants-plugins/release/rules.py +++ b/pants-plugins/release/rules.py @@ -21,6 +21,7 @@ from __future__ import annotations import re +from dataclasses import dataclass from pants.backend.python.util_rules.package_dists import ( SetupKwargs, @@ -88,6 +89,40 @@ def is_applicable(cls, _: Target) -> bool: # return target.address.spec.startswith("st2") +@dataclass(frozen=True) +class StackStormVersionRequest: + version_file: str + description_of_origin: str + + +@dataclass(frozen=True) +class StackStormVersion: + value: str + + +@rule +async def extract_version(request: StackStormVersionRequest) -> StackStormVersion: + version_digest_contents = await Get( + DigestContents, + PathGlobs( + [request.version_file], + description_of_origin=request.description_of_origin, + glob_match_error_behavior=GlobMatchErrorBehavior.error, + ), + ) + + version_file_contents = version_digest_contents[0].content.decode() + version_match = re.search( + r"^__version__ = ['\"]([^'\"]*)['\"]", version_file_contents, re.M + ) + if not version_match: + raise ValueError( + f"Could not find the __version__ in {request.version_file}\n{version_file_contents}" + ) + + return StackStormVersion(version_match.group(1)) + + @rule async def setup_kwargs_plugin(request: StackStormSetupKwargsRequest) -> SetupKwargs: kwargs = request.explicit_kwargs.copy() @@ -100,13 +135,12 @@ async def setup_kwargs_plugin(request: StackStormSetupKwargsRequest) -> SetupKwa version_file = kwargs.pop("version_file") - version_digest_contents, readme_digest_contents = await MultiGet( + version, readme_digest_contents = await MultiGet( Get( - DigestContents, - PathGlobs( - [f"{request.target.address.spec_path}/{version_file}"], + StackStormVersion, + StackStormVersionRequest( + version_file=f"{request.target.address.spec_path}/{version_file}", description_of_origin=f"StackStorm version file: {version_file}", - glob_match_error_behavior=GlobMatchErrorBehavior.error, ), ), Get( @@ -118,19 +152,10 @@ async def setup_kwargs_plugin(request: StackStormSetupKwargsRequest) -> SetupKwa ), ) - version_file_contents = version_digest_contents[0].content.decode() - version_match = re.search( - r"^__version__ = ['\"]([^'\"]*)['\"]", version_file_contents, re.M - ) - if not version_match: - raise ValueError( - f"Could not find the __version__ in {request.target.address.spec_path}/{version_file}\n{version_file_contents}" - ) - # Hardcode certain kwargs and validate that they weren't already set. hardcoded_kwargs = PROJECT_METADATA.copy() hardcoded_kwargs["project_urls"] = FrozenDict(PROJECT_URLS) - hardcoded_kwargs["version"] = version_match.group(1) + hardcoded_kwargs["version"] = version.value long_description = ( readme_digest_contents[0].content.decode() if readme_digest_contents else "" From 85515982a6a19eacc27e2a31ede590ef11c11d40 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Jan 2025 21:30:08 -0600 Subject: [PATCH 55/60] pants-plugins/release: Inject version for nfpm packages --- .pants.bootstrap | 10 ---------- packaging/BUILD | 8 +------- pants-plugins/release/rules.py | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 17 deletions(-) delete mode 100644 .pants.bootstrap diff --git a/.pants.bootstrap b/.pants.bootstrap deleted file mode 100644 index 494bd6c5b9..0000000000 --- a/.pants.bootstrap +++ /dev/null @@ -1,10 +0,0 @@ -# This file should be extremely light-weight. -# Pants sources this file with something like: -# /usr/bin/env bash -c 'set -eou pipefail; source .pants.bootstrap; exec pants' -# Relevant docs: -# https://www.pantsbuild.org/blog/2024/04/27/simple-versioning-with-git-tags -# https://www.pantsbuild.org/stable/docs/using-pants/key-concepts/options#pantsbootstrap-file -# https://github.com/pantsbuild/scie-pants/blob/main/src/main.rs - -# This uses " as the cut delimiter because fmt with black uses " for strings. -export ST2PKG_VERSION="$(grep __version__ st2common/st2common/__init__.py | cut -d'"' -f2)" diff --git a/packaging/BUILD b/packaging/BUILD index d04af10357..ca562d7e60 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -5,10 +5,6 @@ python_sources() # - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package # - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ -_st2_version = env("ST2PKG_VERSION") or "" # from .pants.bootstrap -# semver needs a dash before prerelease, so convert: 3.9dev => 3.9-dev -_st2_version = _st2_version.replace("dev", "-dev") - _pkg_description = """ StackStorm Event-driven automation Package is full standalone st2 installation including all components @@ -20,9 +16,7 @@ _common_pkg_metadata = dict( homepage="https://stackstorm.com", # https://jfearn.fedorapeople.org/en-US/RPM/4/html-single/RPM_Guide/index.html#idp3030720 license="Apache-2.0", # TODO: nFPM is putting this under Copyright tag instead of License - version_schema="semver", - version=_st2_version, # parsed into version[-version_prerelease][+version_metadata] - # TODO: version_release + version="", # injected by pants-plugins/release # arch used to be "any", but that was not correct as the venv has compiled packages. arch="amd64", # TODO: parametrize this? platform="linux", diff --git a/pants-plugins/release/rules.py b/pants-plugins/release/rules.py index 2bdc6931eb..1e06163f93 100644 --- a/pants-plugins/release/rules.py +++ b/pants-plugins/release/rules.py @@ -23,11 +23,14 @@ import re from dataclasses import dataclass +from pants.backend.nfpm.fields.version import NfpmVersionField, NfpmVersionSchemaField +from pants.backend.nfpm.util_rules.inject_config import InjectedNfpmPackageFields, InjectNfpmPackageFieldsRequest from pants.backend.python.util_rules.package_dists import ( SetupKwargs, SetupKwargsRequest, ) from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs +from pants.engine.internals.native_engine import Field from pants.engine.target import Target from pants.engine.rules import collect_rules, Get, MultiGet, rule, UnionRule from pants.util.frozendict import FrozenDict @@ -187,8 +190,41 @@ async def setup_kwargs_plugin(request: StackStormSetupKwargsRequest) -> SetupKwa return SetupKwargs(kwargs, address=request.target.address) +class StackStormNfpmPackageFieldsRequest(InjectNfpmPackageFieldsRequest): + @classmethod + def is_applicable(cls, _: Target) -> bool: + return True + + +@rule +async def inject_package_fields(request: StackStormNfpmPackageFieldsRequest) -> InjectedNfpmPackageFields: + address = request.target.address + + version_file = "st2common/st2common/__init__.py" + extracted_version = await Get( + StackStormVersion, + StackStormVersionRequest( + version_file=version_file, + description_of_origin=f"StackStorm version file: {version_file}", + ), + ) + + version: str = extracted_version.value + if version.endswith("dev") and version[-4] != "-": + # nfpm parses this into version[-version_prerelease][+version_metadata] + # that dash is required to be a valid semver version. + version = version.replace("dev", "-dev") + + fields: list[Field] = [ + NfpmVersionSchemaField("semver", address=address), + NfpmVersionField(version, address=address), + ] + return InjectedNfpmPackageFields(fields, address=address) + + def rules(): return [ *collect_rules(), UnionRule(SetupKwargsRequest, StackStormSetupKwargsRequest), + UnionRule(InjectNfpmPackageFieldsRequest, StackStormNfpmPackageFieldsRequest), ] From b2233daddb846bbe5191098e1dd2dbfc939b1c5a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 9 Jan 2025 21:34:29 -0600 Subject: [PATCH 56/60] pants-plugins/macros: typo fix --- pants-plugins/macros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 0f5a631560..b30b4a1b09 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -254,6 +254,6 @@ def st2_logging_conf_for_nfpm(**kwargs): file_owner="root", file_group="root", file_mode="rw-r--r--", - content_type="config|noreplace" + content_type="config|noreplace", **kwargs, ) From 81218560b754cfe53c934a3f7912ce9d3ca17bad Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 10 Jan 2025 14:47:47 -0600 Subject: [PATCH 57/60] pants: bump to pants 2.23.1 and add requests for pants-plugins pants-plugins/release will need requests to call packagecloud APIs. Lockfile diff: lockfiles/pants-plugins.lock [pants-plugins] == Upgraded dependencies == attrs 24.2.0 --> 24.3.0 pantsbuild-pants 2.23.0a0 --> 2.23.1 pantsbuild-pants-testutil 2.23.0a0 --> 2.23.1 pyparsing 3.1.4 --> 3.2.1 six 1.16.0 --> 1.17.0 tomli 2.0.1 --> 2.2.1 == Added dependencies == certifi 2024.12.14 charset-normalizer 3.4.1 idna 3.10 requests 2.32.3 urllib3 2.3.0 --- lockfiles/pants-plugins.lock | 266 ++++++++++++++++++++----- pants-plugins/release/BUILD | 4 + pants-plugins/release/requirements.txt | 1 + pants.toml | 2 +- 4 files changed, 221 insertions(+), 52 deletions(-) create mode 100644 pants-plugins/release/requirements.txt diff --git a/lockfiles/pants-plugins.lock b/lockfiles/pants-plugins.lock index 4d9a43a8aa..531542521b 100644 --- a/lockfiles/pants-plugins.lock +++ b/lockfiles/pants-plugins.lock @@ -9,8 +9,9 @@ // "CPython==3.9.*" // ], // "generated_with_requirements": [ -// "pantsbuild.pants.testutil==2.23.0a0", -// "pantsbuild.pants==2.23.0a0" +// "pantsbuild.pants.testutil==2.23.1", +// "pantsbuild.pants==2.23.1", +// "requests" // ], // "manylinux": "manylinux2014", // "requirement_constraints": [], @@ -51,13 +52,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2", - "url": "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl" + "hash": "ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", + "url": "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "url": "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz" + "hash": "8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", + "url": "https://files.pythonhosted.org/packages/48/c8/6260f8ccc11f0917360fc0da435c5c9c7504e3db174d5a12a1494887b045/attrs-24.3.0.tar.gz" } ], "project_name": "attrs", @@ -73,24 +74,23 @@ "hypothesis; extra == \"cov\"", "hypothesis; extra == \"dev\"", "hypothesis; extra == \"tests\"", - "importlib-metadata; python_version < \"3.8\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"benchmark\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"cov\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"dev\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"tests\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"tests-mypy\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"benchmark\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"cov\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"dev\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests-mypy\"", "myst-parser; extra == \"docs\"", - "pre-commit; extra == \"dev\"", + "pre-commit-uv; extra == \"dev\"", "pympler; extra == \"benchmark\"", "pympler; extra == \"cov\"", "pympler; extra == \"dev\"", "pympler; extra == \"tests\"", "pytest-codspeed; extra == \"benchmark\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"benchmark\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"cov\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"dev\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"tests\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"tests-mypy\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"benchmark\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"cov\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"dev\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests-mypy\"", "pytest-xdist[psutil]; extra == \"benchmark\"", "pytest-xdist[psutil]; extra == \"cov\"", "pytest-xdist[psutil]; extra == \"dev\"", @@ -104,8 +104,99 @@ "sphinxcontrib-towncrier; extra == \"docs\"", "towncrier<24.7; extra == \"docs\"" ], + "requires_python": ">=3.8", + "version": "24.3.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "url": "https://files.pythonhosted.org/packages/a5/32/8f6669fc4798494966bf446c8c4a162e0b5d893dff088afddf76414f70e1/certifi-2024.12.14-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db", + "url": "https://files.pythonhosted.org/packages/0f/bd/1d41ee578ce09523c81a15426705dd20969f5abf006d1afe8aeff0dd776a/certifi-2024.12.14.tar.gz" + } + ], + "project_name": "certifi", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2024.12.14" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", + "url": "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496", + "url": "https://files.pythonhosted.org/packages/04/d2/42fd330901aaa4b805a1097856c2edf5095e260a597f65def493f4b8c833/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f", + "url": "https://files.pythonhosted.org/packages/0f/6c/2bee440303d705b6fb1e2ec789543edec83d32d258299b16eed28aad48e0/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", + "url": "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7", + "url": "https://files.pythonhosted.org/packages/26/ae/23d6041322a3556e4da139663d02fb1b3c59a23ab2e2b56432bd2ad63ded/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2", + "url": "https://files.pythonhosted.org/packages/3d/04/cb42585f07f6f9fd3219ffb6f37d5a39b4fd2db2355b23683060029c35f7/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770", + "url": "https://files.pythonhosted.org/packages/54/54/2412a5b093acb17f0222de007cc129ec0e0df198b5ad2ce5699355269dfe/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4", + "url": "https://files.pythonhosted.org/packages/5a/6d/e2773862b043dcf8a221342954f375392bb2ce6487bcd9f2c1b34e1d6781/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41", + "url": "https://files.pythonhosted.org/packages/7f/c0/b913f8f02836ed9ab32ea643c6fe4d3325c3d8627cf6e78098671cafff86/charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6", + "url": "https://files.pythonhosted.org/packages/94/22/b8f2081c6a77cb20d97e57e0b385b481887aa08019d2459dc2858ed64871/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78", + "url": "https://files.pythonhosted.org/packages/9e/af/3a97a4fa3c53586f1910dadfc916e9c4f35eeada36de4108f5096cb7215f/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537", + "url": "https://files.pythonhosted.org/packages/b9/f8/ca440ef60d8f8916022859885f231abb07ada3c347c03d63f283bec32ef5/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294", + "url": "https://files.pythonhosted.org/packages/c7/0b/c5ec5092747f801b8b093cdf5610e732b809d6cb11f4c51e35fc28d1d389/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [], "requires_python": ">=3.7", - "version": "24.2.0" + "version": "3.4.1" }, { "artifacts": [ @@ -146,6 +237,29 @@ "requires_python": null, "version": "0.16.3" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", + "url": "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "url": "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz" + } + ], + "project_name": "idna", + "requires_dists": [ + "flake8>=7.1.1; extra == \"all\"", + "mypy>=1.11.2; extra == \"all\"", + "pytest>=8.3.2; extra == \"all\"", + "ruff>=0.6.2; extra == \"all\"" + ], + "requires_python": ">=3.6", + "version": "3.10" + }, { "artifacts": [ { @@ -346,23 +460,23 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "f7104cf619c928752041acfe36966742dec5309b171aeef921239d4595ee4161", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-manylinux2014_x86_64.whl" + "hash": "4f4f7a4e35ae7562680722de03365ea31b4ff929996e81ee7994f931563b3095", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.1/pantsbuild.pants-2.23.1-cp39-cp39-manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "d74b12dd7c4dd4cc9a7c81e55126db298577830c62962a6f8cbb4d875930f9ed", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-macosx_10_15_x86_64.whl" + "hash": "465a7b14d2a7862e5c4d688478bb378afcff17359391ec9543dfe0b282422640", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.1/pantsbuild.pants-2.23.1-cp39-cp39-macosx_10_15_x86_64.whl" }, { "algorithm": "sha256", - "hash": "3afee18ce33b16cb3147ed18e190f0e37d4f3561d58354ee1203f7c66cfe1c5f", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "231d4b2aff884b3b7e3c6ab53cc51b88a7fbd50a91e1add5ccae31750c3a680c", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.1/pantsbuild.pants-2.23.1-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "6e47e4076e8321005b15afa4bd63f1444e32446de2634043caeafa35853a279c", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-manylinux2014_aarch64.whl" + "hash": "88881c1932bcbb9c69818773afe268b53239d4a00bbbef7116bd6f5c8c4abfb2", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.1/pantsbuild.pants-2.23.1-cp39-cp39-manylinux2014_aarch64.whl" } ], "project_name": "pantsbuild-pants", @@ -387,23 +501,23 @@ "typing-extensions~=4.12" ], "requires_python": "==3.9.*", - "version": "2.23.0a0" + "version": "2.23.1" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "f74af1d1cbac2f8c17e441e2e6c96588fc1816828ecc2665b535dd4ccfbaa6c7", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants.testutil-2.23.0a0-py3-none-any.whl" + "hash": "5800cdcb78a854bd87a9f6d1abbd2b89e4df7e5c2a1ce28f55f5dfc59b1f6d9e", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.1/pantsbuild.pants.testutil-2.23.1-py3-none-any.whl" } ], "project_name": "pantsbuild-pants-testutil", "requires_dists": [ - "pantsbuild.pants==2.23.0a0", + "pantsbuild.pants==2.23.1", "pytest<7.1.0,>=6.2.4" ], "requires_python": "==3.9.*", - "version": "2.23.0a0" + "version": "2.23.1" }, { "artifacts": [ @@ -509,13 +623,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", - "url": "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl" + "hash": "506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1", + "url": "https://files.pythonhosted.org/packages/1c/a7/c8a2d361bf89c0d9577c934ebb7421b25dc84bf3a8e3ac0a40aed9acc547/pyparsing-3.2.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032", - "url": "https://files.pythonhosted.org/packages/83/08/13f3bce01b2061f2bbd582c9df82723de943784cf719a35ac886c652043a/pyparsing-3.1.4.tar.gz" + "hash": "61980854fd66de3a90028d679a954d5f2623e83144b5afe5ee86f43d762e5f0a", + "url": "https://files.pythonhosted.org/packages/8b/1a/3544f4f299a47911c2ab3710f534e52fea62a633c96806995da5d25be4b2/pyparsing-3.2.1.tar.gz" } ], "project_name": "pyparsing", @@ -523,8 +637,8 @@ "jinja2; extra == \"diagrams\"", "railroad-diagrams; extra == \"diagrams\"" ], - "requires_python": ">=3.6.8", - "version": "3.1.4" + "requires_python": ">=3.9", + "version": "3.2.1" }, { "artifacts": [ @@ -635,6 +749,31 @@ "requires_python": ">=3.8", "version": "6.0.2" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", + "url": "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "url": "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz" + } + ], + "project_name": "requests", + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"", + "certifi>=2017.4.17", + "chardet<6,>=3.0.2; extra == \"use-chardet-on-py3\"", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1" + ], + "requires_python": ">=3.8", + "version": "2.32.3" + }, { "artifacts": [ { @@ -782,19 +921,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", - "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" + "hash": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "url": "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "url": "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" + "hash": "ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", + "url": "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz" } ], "project_name": "six", "requires_dists": [], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", - "version": "1.16.0" + "version": "1.17.0" }, { "artifacts": [ @@ -818,19 +957,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl" + "hash": "cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", + "url": "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f", - "url": "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz" + "hash": "cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", + "url": "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz" } ], "project_name": "tomli", "requires_dists": [], - "requires_python": ">=3.7", - "version": "2.0.1" + "requires_python": ">=3.8", + "version": "2.2.1" }, { "artifacts": [ @@ -981,6 +1120,30 @@ "requires_dists": [], "requires_python": ">=3.8", "version": "5.10.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", + "url": "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d", + "url": "https://files.pythonhosted.org/packages/aa/63/e53da845320b757bf29ef6a9062f5c669fe997973f966045cb019c3f4b66/urllib3-2.3.0.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", + "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", + "h2<5,>=4; extra == \"h2\"", + "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "zstandard>=0.18.0; extra == \"zstd\"" + ], + "requires_python": ">=3.9", + "version": "2.3.0" } ], "platform_tag": null @@ -994,8 +1157,9 @@ "pip_version": "24.0", "prefer_older_binary": false, "requirements": [ - "pantsbuild.pants.testutil==2.23.0a0", - "pantsbuild.pants==2.23.0a0" + "pantsbuild.pants.testutil==2.23.1", + "pantsbuild.pants==2.23.1", + "requests" ], "requires_python": [ "==3.9.*" diff --git a/pants-plugins/release/BUILD b/pants-plugins/release/BUILD index 0eea8b1cf1..c485368c66 100644 --- a/pants-plugins/release/BUILD +++ b/pants-plugins/release/BUILD @@ -3,3 +3,7 @@ python_sources() python_tests( name="tests", ) + +python_requirements( + name="reqs", +) diff --git a/pants-plugins/release/requirements.txt b/pants-plugins/release/requirements.txt new file mode 100644 index 0000000000..f2293605cf --- /dev/null +++ b/pants-plugins/release/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/pants.toml b/pants.toml index e4c496f0c0..1c72755d85 100644 --- a/pants.toml +++ b/pants.toml @@ -6,7 +6,7 @@ enabled = false repo_id = "de0dea7a-9f6a-4c6e-aa20-6ba5ad969b8a" [GLOBAL] -pants_version = "2.23.0a0" +pants_version = "2.23.1" pythonpath = ["%(buildroot)s/pants-plugins"] build_file_prelude_globs = ["pants-plugins/macros.py"] backend_packages = [ From 1d27f08b75c1e21d15b484beba565de5faf58810 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 10 Jan 2025 16:54:48 -0600 Subject: [PATCH 58/60] pants-plugins/macros: fmt --- pants-plugins/macros.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index b30b4a1b09..b6969d90e2 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -177,7 +177,7 @@ def st2_pack_archive(**kwargs): ) nfpm_content_file( # noqa: F821 - name=f"archive_for_nfpm", + name="archive_for_nfpm", dependencies=[":archive"], src=f"packaging/packs/{pack_name}.tgz.run", dst=f"/opt/stackstorm/install/packs/{pack_name}.tgz.run", @@ -236,7 +236,7 @@ def st2_logging_conf_resources(**kwargs): def st2_logging_conf_for_nfpm(**kwargs): deps = kwargs.pop("dependencies") or [] - shell_command( + shell_command( # noqa: F821 name="package_logging_conf", execution_dependencies=deps, # TODO: this will fail if using MacOS. @@ -248,7 +248,7 @@ def st2_logging_conf_for_nfpm(**kwargs): output_files=["*.conf"], ) - nfpm_content_files( + nfpm_content_files( # noqa: F821 name="packaged_conf_files", dependencies=[":package_logging_conf"], file_owner="root", From 945b0b3c47680f3fdfecc82b4e34e6261a40f61c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 10 Jan 2025 17:23:02 -0600 Subject: [PATCH 59/60] pants-plugins/release: calculate packagecloud next release number --- pants-plugins/release/packagecloud_rules.py | 155 ++++++++++++++++++++ pants-plugins/release/rules.py | 47 +++++- 2 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 pants-plugins/release/packagecloud_rules.py diff --git a/pants-plugins/release/packagecloud_rules.py b/pants-plugins/release/packagecloud_rules.py new file mode 100644 index 0000000000..bdda2399d8 --- /dev/null +++ b/pants-plugins/release/packagecloud_rules.py @@ -0,0 +1,155 @@ +# Copyright 2025 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import requests +from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest +from requests.auth import HTTPBasicAuth + +from pants.engine.internals.selectors import Get +from pants.engine.rules import _uncacheable_rule, collect_rules + +ARCH_NAMES = { # {nfpm_arch: {pkg_type: packagecloud_arch}} + # The key comes from the 'arch' field of nfpm_*_package targets (GOARCH or GOARCH+GOARM). + # https://www.pantsbuild.org/stable/reference/targets/nfpm_deb_package#arch + # https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package#arch + "amd64": { + "deb": "amd64", + "rpm": "x86_64", + } +} + +# This includes distros we do not support. +DISTROS_BY_PKG_TYPE = { # {pkg_type: {distro: {distro_id: distro_version}}} + "deb": { + "debian": { # no releases in packagecloud (so far) + "buster": "10", + "bullseye": "11", + "bookworm": "12", + "trixie": "13", + "forky": "14", + }, + "ubuntu": { # Only LTS releases + "trusty": "14.04", # the oldest with releases in packagecloud + "xenial": "16.04", + "bionic": "18.04", + "focal": "20.04", + "jammy": "22.04", + "noble": "24.04", + }, + }, + "rpm": { + "el": { # EL = Enterprise Linux (RHEL, Rocky, Alma, ...) + # 6 is the oldest with releases in packagecloud + f"el{v}": f"{v}" + for v in (6, 7, 8, 9) + }, + }, +} + +DISTRO_INFO = { + distro_id: { + "distro": distro, + "version": distro_version, + "pkg_type": pkg_type, + } + for pkg_type, distros in DISTROS_BY_PKG_TYPE.items() + for distro, distro_ids in distros.items() + for distro_id, distro_version in distro_ids.items() +} + + +@dataclass +class PackageCloudNextReleaseRequest: + nfpm_arch: str + distro_id: str + package_name: str + package_version: str + production: bool + + +@dataclass +class PackageCloudNextRelease: + value: Optional[int] = None + + +@_uncacheable_rule +async def packagecloud_get_next_release( + request: PackageCloudNextReleaseRequest, +) -> PackageCloudNextRelease: + env_vars: EnvironmentVars = await Get( + EnvironmentVars, EnvironmentVarsRequest(["PACKAGECLOUD_TOKEN"]) + ) + package_cloud_token = env_vars.get("PACKAGECLOUD_TOKEN") + if not package_cloud_token: + return PackageCloudNextRelease() + + client = requests.session() + client.auth = HTTPBasicAuth(package_cloud_token, "") + + def get(url_path: str) -> list[dict[str, Any]]: + response = client.get(f"https://packagecloud.io{url_path}") + response.raise_for_status() + ret: list[dict[str, Any]] = response.json() + next_url = response.links.get("next", {}).get("url") + while next_url: + response = client.get(f"https://packagecloud.io{next_url}") + response.raise_for_status() + ret.extend(response.json()) + next_url = response.links.get("next", {}).get("url") + return ret + + distro_id = request.distro_id + distro_info = DISTRO_INFO[distro_id] + pkg_is_unstable = "dev" in request.package_version + + # packagecloud url params: + org = "stackstorm" + repo = f"{'' if request.production else 'staging-'}{'unstable' if pkg_is_unstable else 'stable'}" + pkg_type = distro_info["pkg_type"] + distro = distro_info["distro"] + distro_version = distro_id if pkg_type == "deb" else distro_info["version"] + pkg_name = request.package_name + arch = ARCH_NAMES[request.nfpm_arch][pkg_type] + + # https://packagecloud.io/docs/api#resource_packages_method_index (api doc incorrectly drops /:package) + # /api/v1/repos/:user_id/:repo/packages/:type/:distro/:version/:package/:arch.json + index_url = f"/api/v1/repos/{org}/{repo}/packages/{pkg_type}/{distro}/{distro_version}/{pkg_name}/{arch}.json" + package_index: list[dict[str, Any]] = get(index_url) + if not package_index: + return PackageCloudNextRelease() + + versions_url: str = package_index[0]["versions_url"] + versions: list[dict[str, Any]] = get(versions_url) + releases = [ + version_info["release"] + for version_info in versions + if version_info["version"] == request.package_version + ] + if not releases: + return PackageCloudNextRelease() + + max_release = max(int(release) for release in releases) + next_release = max_release + 1 + return PackageCloudNextRelease(next_release) + + +def rules(): + return [ + *collect_rules(), + ] diff --git a/pants-plugins/release/rules.py b/pants-plugins/release/rules.py index 1e06163f93..349729adf4 100644 --- a/pants-plugins/release/rules.py +++ b/pants-plugins/release/rules.py @@ -23,8 +23,19 @@ import re from dataclasses import dataclass -from pants.backend.nfpm.fields.version import NfpmVersionField, NfpmVersionSchemaField -from pants.backend.nfpm.util_rules.inject_config import InjectedNfpmPackageFields, InjectNfpmPackageFieldsRequest +from nfpm.fields.all import ( + NfpmArchField, + NfpmPackageNameField, +) +from nfpm.fields.version import ( + NfpmVersionField, + NfpmVersionReleaseField, + NfpmVersionSchemaField, +) +from nfpm.util_rules.inject_config import ( + InjectedNfpmPackageFields, + InjectNfpmPackageFieldsRequest, +) from pants.backend.python.util_rules.package_dists import ( SetupKwargs, SetupKwargsRequest, @@ -35,6 +46,12 @@ from pants.engine.rules import collect_rules, Get, MultiGet, rule, UnionRule from pants.util.frozendict import FrozenDict +from .packagecloud_rules import ( + PackageCloudNextReleaseRequest, + packagecloud_get_next_release, +) +from .packagecloud_rules import rules as packagecloud_rules + REQUIRED_KWARGS = ( "description", @@ -197,8 +214,11 @@ def is_applicable(cls, _: Target) -> bool: @rule -async def inject_package_fields(request: StackStormNfpmPackageFieldsRequest) -> InjectedNfpmPackageFields: - address = request.target.address +async def inject_package_fields( + request: StackStormNfpmPackageFieldsRequest, +) -> InjectedNfpmPackageFields: + target = request.target + address = target.address version_file = "st2common/st2common/__init__.py" extracted_version = await Get( @@ -210,20 +230,35 @@ async def inject_package_fields(request: StackStormNfpmPackageFieldsRequest) -> ) version: str = extracted_version.value - if version.endswith("dev") and version[-4] != "-": + is_dev = "dev" in version + if is_dev and "-dev" not in version: # nfpm parses this into version[-version_prerelease][+version_metadata] - # that dash is required to be a valid semver version. + # that dash is required to be a valid semver version (3.9dev => 3.9-dev). version = version.replace("dev", "-dev") + # this is specific to distro-version (EL8, EL9, Ubuntu Focal, Ubuntu Jammy, ...) + next_release = await packagecloud_get_next_release( + PackageCloudNextReleaseRequest( + nfpm_arch=target[NfpmArchField].value, + distro_id="", # TODO: add field for distro ID + package_name=target[NfpmPackageNameField].value, + package_version=version, + production=not is_dev, + ) + ) + release = 1 if next_release.value is None else next_release.value + fields: list[Field] = [ NfpmVersionSchemaField("semver", address=address), NfpmVersionField(version, address=address), + NfpmVersionReleaseField(release, address=address), ] return InjectedNfpmPackageFields(fields, address=address) def rules(): return [ + *packagecloud_rules(), *collect_rules(), UnionRule(SetupKwargsRequest, StackStormSetupKwargsRequest), UnionRule(InjectNfpmPackageFieldsRequest, StackStormNfpmPackageFieldsRequest), From bf70cae10a2e68cf0fd970ea07e1726d173bb19b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 13 Jan 2025 10:23:36 -0600 Subject: [PATCH 60/60] pants-plugins/release: add distro_id field to nfpm rpm/deb package targets This should also be included in the output_path and possibly other fields. --- packaging/BUILD | 2 + pants-plugins/release/register.py | 6 ++- pants-plugins/release/rules.py | 3 +- pants-plugins/release/target_types.py | 53 +++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 pants-plugins/release/target_types.py diff --git a/packaging/BUILD b/packaging/BUILD index ca562d7e60..fb4873b19b 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -25,6 +25,7 @@ _maintainer = "StackStorm Engineering " # TODO: update nfpm_deb_package( name="st2.deb", + distro_id=parametrize("focal"), dependencies=[ "./deb/scripts", "./deb/systemd:services", @@ -95,6 +96,7 @@ nfpm_deb_package( nfpm_rpm_package( name="st2.rpm", + distro_id=parametrize("el8", "el9"), dependencies=[ "./rpm/scripts", "./rpm/systemd:services", diff --git a/pants-plugins/release/register.py b/pants-plugins/release/register.py index b3fa04132f..93a680a563 100644 --- a/pants-plugins/release/register.py +++ b/pants-plugins/release/register.py @@ -17,7 +17,11 @@ """ from release.rules import rules as release_rules +from release.target_types import rules as release_target_types_rules def rules(): - return release_rules() + return [ + *release_target_types_rules(), + *release_rules(), + ] diff --git a/pants-plugins/release/rules.py b/pants-plugins/release/rules.py index 349729adf4..f009c696f1 100644 --- a/pants-plugins/release/rules.py +++ b/pants-plugins/release/rules.py @@ -51,6 +51,7 @@ packagecloud_get_next_release, ) from .packagecloud_rules import rules as packagecloud_rules +from .target_types import DistroIDField REQUIRED_KWARGS = ( @@ -240,7 +241,7 @@ async def inject_package_fields( next_release = await packagecloud_get_next_release( PackageCloudNextReleaseRequest( nfpm_arch=target[NfpmArchField].value, - distro_id="", # TODO: add field for distro ID + distro_id=target[DistroIDField].value, package_name=target[NfpmPackageNameField].value, package_version=version, production=not is_dev, diff --git a/pants-plugins/release/target_types.py b/pants-plugins/release/target_types.py new file mode 100644 index 0000000000..1b196fc980 --- /dev/null +++ b/pants-plugins/release/target_types.py @@ -0,0 +1,53 @@ +# Copyright 2025 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pants.engine.target import StringField +from pants.util.strutil import help_text + +from nfpm.target_types import NfpmDebPackage, NfpmRpmPackage + + +class DistroIDField(StringField): + nfpm_alias = "" # Not an nFPM field + alias = "distro_id" + valid_choices = ( # officially supported (or planned future support) + # ubuntu + "focal", + "jammy", + "noble", + # el + "el8", + "el9", + ) + required = True + help = help_text( + f""" + The package distribution and version. + + This is an internal StackStorm field used by pants-plugins/release. + The IDs are StackStorm-specific IDs that get translated into distribution + version. + These examples show how the distro_id gets translated into packagecloud values: + - distro_id "el8" is distro "el" with version "8"; + - distro_id "focal" is distro "ubuntu" with version "focal". + """ + ) + + +def rules(): + return [ + NfpmDebPackage.register_plugin_field(DistroIDField), + NfpmRpmPackage.register_plugin_field(DistroIDField), + ] \ No newline at end of file