diff --git a/.asf.yaml b/.asf.yaml index a19e3a60a2ba..df0dc3ab56c7 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -50,10 +50,14 @@ github: rebase: false collaborators: - - gpordeus + - ingox + - gp-santos - erikbocks - Imvedansh - Damans227 + - jmsperu + - GaOrtiga + - bhouse-nexthop protected_branches: ~ diff --git a/.github/actions/install-nonoss/action.yml b/.github/actions/install-nonoss/action.yml new file mode 100644 index 000000000000..39a03213c29d --- /dev/null +++ b/.github/actions/install-nonoss/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Install CloudStack Non-OSS' +description: 'Clones and installs the shapeblue/cloudstack-nonoss repository.' + +runs: + using: "composite" + steps: + - name: Install cloudstack-nonoss + shell: bash + run: | + git clone --depth 1 https://github.com/shapeblue/cloudstack-nonoss.git nonoss + cd nonoss + bash -x install-non-oss.sh + cd .. + rm -fr nonoss diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 000000000000..0f8425229242 --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup CloudStack Environment' +description: 'Sets up JDK (with Maven cache), optionally Python, and optionally APT build dependencies for CloudStack.' + +inputs: + java-version: + description: 'The JDK version to use' + required: false + default: '17' + install-python: + description: 'Whether to install Python 3.10' + required: false + default: 'false' + install-apt-deps: + description: 'Whether to install CloudStack APT build dependencies' + required: false + default: 'false' + +runs: + using: "composite" + steps: + - name: Set up JDK ${{ inputs.java-version }} + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: ${{ inputs.java-version }} + distribution: 'adopt' + architecture: x64 + cache: 'maven' + + - name: Set up Python + if: ${{ inputs.install-python == 'true' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10' + architecture: x64 + + - name: Install Build Dependencies + if: ${{ inputs.install-apt-deps == 'true' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 000000000000..ea25ffab6b9e --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,56 @@ +{ + "entries": { + "actions/github-script@v9.0.0": { + "repo": "actions/github-script", + "version": "v9.0.0", + "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" + }, + "github/gh-aw-actions/setup@v0.76.1": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.76.1", + "sha": "46d564922b082d0db93244972e8005ea6904ee5f" + } + }, + "containers": { + "ghcr.io/github/gh-aw-firewall/agent:0.18.0": { + "image": "ghcr.io/github/gh-aw-firewall/agent:0.18.0", + "digest": "sha256:ab84dfc7f5998cb8cd0c596526dd573b7e7d06c6a740266a1e6df879fa16c866", + "pinned_image": "ghcr.io/github/gh-aw-firewall/agent:0.18.0@sha256:ab84dfc7f5998cb8cd0c596526dd573b7e7d06c6a740266a1e6df879fa16c866" + }, + "ghcr.io/github/gh-aw-firewall/agent:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/agent:0.25.55", + "digest": "sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731", + "pinned_image": "ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731" + }, + "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55", + "digest": "sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3", + "pinned_image": "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3" + }, + "ghcr.io/github/gh-aw-firewall/squid:0.18.0": { + "image": "ghcr.io/github/gh-aw-firewall/squid:0.18.0", + "digest": "sha256:82a5d062a5612a57a43a171a5b79ddbb690a86a8ddda02339cc1675131ae9f8b", + "pinned_image": "ghcr.io/github/gh-aw-firewall/squid:0.18.0@sha256:82a5d062a5612a57a43a171a5b79ddbb690a86a8ddda02339cc1675131ae9f8b" + }, + "ghcr.io/github/gh-aw-firewall/squid:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/squid:0.25.55", + "digest": "sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca", + "pinned_image": "ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca" + }, + "ghcr.io/github/gh-aw-mcpg:v0.1.4": { + "image": "ghcr.io/github/gh-aw-mcpg:v0.1.4", + "digest": "sha256:0acf25aa1d409f9c73be9e39ac84f4bd4b90d8bfa1db4dc6d7f47d38ccd58914", + "pinned_image": "ghcr.io/github/gh-aw-mcpg:v0.1.4@sha256:0acf25aa1d409f9c73be9e39ac84f4bd4b90d8bfa1db4dc6d7f47d38ccd58914" + }, + "ghcr.io/github/gh-aw-mcpg:v0.3.19": { + "image": "ghcr.io/github/gh-aw-mcpg:v0.3.19", + "digest": "sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f", + "pinned_image": "ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f" + }, + "ghcr.io/github/github-mcp-server:v0.30.3": { + "image": "ghcr.io/github/github-mcp-server:v0.30.3", + "digest": "sha256:a2b5fb79b1cee851bfc3532dfe480c3dc5736974ca9d93a7a9f68e52ce4b62a0", + "pinned_image": "ghcr.io/github/github-mcp-server:v0.30.3@sha256:a2b5fb79b1cee851bfc3532dfe480c3dc5736974ca9d93a7a9f68e52ce4b62a0" + } + } +} diff --git a/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_noop-reminder.md b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_noop-reminder.md new file mode 100644 index 000000000000..77cca08c1bec --- /dev/null +++ b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_noop-reminder.md @@ -0,0 +1,5 @@ +**Important**: If no action is needed after completing your analysis, you **MUST** call the `noop` safe-output tool with a brief explanation. Failing to call any safe-output tool is the most common cause of safe-output workflow failures. + +```json +{"noop": {"message": "No action needed: [brief explanation of what was analyzed and why]"}} +``` diff --git a/.github/aw/imports/github/gh-aw/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github_workflows_shared_reporting.md b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md similarity index 85% rename from .github/aw/imports/github/gh-aw/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github_workflows_shared_reporting.md rename to .github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md index bc08afb42be9..72d61b434efa 100644 --- a/.github/aw/imports/github/gh-aw/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github_workflows_shared_reporting.md +++ b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md @@ -13,7 +13,7 @@ When creating GitHub issues or discussions: - Never use `##` (h2) or `#` (h1) in reports - these are reserved for titles ### 2. Progressive Disclosure -**Wrap detailed test results in `
Section Name` tags to improve readability and reduce scrolling.** +**Wrap detailed test results in `
Section Name` tags to improve readability and reduce scrolling.** Use collapsible sections for: - Verbose details (full test logs, raw data) @@ -26,7 +26,7 @@ Always keep critical information visible (summary, critical issues, key metrics) 1. **Overview**: 1-2 paragraphs summarizing key findings 2. **Critical Information**: Show immediately (summary stats, critical issues) -3. **Details**: Use `
Section Name` for expanded content +3. **Details**: Use `
Section Name` for expanded content 4. **Context**: Add helpful metadata (workflow run, date, trigger) ### Design Principles (Airbnb-Inspired) @@ -49,14 +49,14 @@ Reports should: [Always visible - these are important]
-View Detailed Results +View Detailed Results [Comprehensive details, logs, traces]
-View All Warnings +View All Warnings [Minor issues and potential problems] diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cef74aafb4b0..6ffb926f6fa0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -31,6 +31,8 @@ updates: github-actions-dependencies: patterns: - "*" + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. cooldown: default-days: 7 - package-ecosystem: "maven" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c33a1313436..4eb508f5f6f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,40 +16,27 @@ # under the License. name: Build - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v6 + - name: Setup Environment + uses: ./.github/actions/setup-env with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Env details run: | uname -a @@ -60,9 +47,8 @@ jobs: free -m nproc git status - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Noredist Build run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df60179ceb51..0815b4558476 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,21 +16,56 @@ # under the License. name: Simulator CI - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: if: github.repository == 'apache/cloudstack' runs-on: ubuntu-24.04 - + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + with: + install-python: 'true' + install-apt-deps: 'true' + - name: Env details + run: | + uname -a + whoami + javac -version + mvn -v + python3 --version + free -m + nproc + git status + ipmitool -V + - name: Build with Maven + run: | + mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) + - name: Archive artifacts + run: | + mkdir -p /tmp/artifacts + tar -czf /tmp/artifacts/targets.tar.gz $(find . -name "target" -type d) tools/marvin/dist engine/schema/dist utils/conf + tar -czf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository org/apache/cloudstack + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + test: + needs: build + if: github.repository == 'apache/cloudstack' + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -215,30 +250,16 @@ jobs: smoke/test_list_service_offerings smoke/test_list_storage_pools smoke/test_list_volumes"] - steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Setup IPMI Tool for CloudStack run: | # Create cloudstack-common directory if it doesn't exist @@ -256,55 +277,43 @@ jobs: /usr/share/cloudstack-common/ipmitool -C3 $@ EOF sudo chmod 755 /usr/bin/ipmitool - - name: Install Python dependencies run: | python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose - - name: Install jacoco dependencies run: | wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip unzip jacoco-0.8.10.zip -d jacoco - - - name: Env details - run: | - uname -a - whoami - javac -version - mvn -v - python3 --version - free -m - nproc - git status - ipmitool -V - - name: Setup MySQL Server run: | # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#mysql sudo apt-get install -y mysql-server sudo systemctl start mysql - sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; FLUSH PRIVILEGES;" + sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH caching_sha2_password BY ''; FLUSH PRIVILEGES;" sudo systemctl restart mysql sudo mysql -uroot -e "SELECT VERSION();" - - - name: Build with Maven + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + - name: Extract artifacts run: | - mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) - + tar -xzf /tmp/artifacts/targets.tar.gz + mkdir -p ~/.m2/repository + tar -xzf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository - name: Setup Simulator Prerequisites run: | sudo python3 -m pip install --upgrade netaddr mysql-connector-python python3 -m pip install --user --upgrade tools/marvin/dist/[mM]arvin-*.tar.gz mvn -q -Pdeveloper -pl developer -Ddeploydb mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator - - name: Generate jacoco-coverage.sh run: | echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh echo "--xml jacoco-coverage.xml" >> jacoco-report.sh - - name: Start CloudStack Management Server with Simulator run: | export MAVEN_OPTS="-Xmx4096m -XX:MaxMetaspaceSize=800m -Djava.security.egd=file:/dev/urandom -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" @@ -315,7 +324,6 @@ jobs: set -e echo -e "\nStarting Advanced Zone DataCenter deployment" python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true - - name: Run Integration Tests with Simulator run: | mkdir -p integration-test-results/smoke/misc @@ -335,13 +343,12 @@ jobs: bash jacoco-report.sh mvn -Dsimulator -pl client jetty:stop 2>&1 find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \; - - name: Integration Tests Result run: | echo -e "Simulator CI Test Results: (only failures listed)\n" python3 ./tools/marvin/xunit-reader.py integration-test-results/ - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: files: jacoco-coverage.xml fail_ci_if_error: true diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 88b10ac9178f..000000000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Coverage Check - -on: [pull_request, push] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - if: github.repository == 'apache/cloudstack' - name: codecov - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Build CloudStack with Quality Checks - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc) - - - uses: codecov/codecov-action@v4 - with: - files: ./client/target/site/jacoco-aggregate/jacoco.xml - fail_ci_if_error: true - flags: unittests - verbose: true - name: codecov - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 31a8746b85af..a2d581dcac20 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,14 +35,16 @@ jobs: language: ["actions"] steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: category: "Security" diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml index 1d7e7eecd14d..7c3d20a166ea 100644 --- a/.github/workflows/daily-repo-status.lock.yml +++ b/.github/workflows/daily-repo-status.lock.yml @@ -1,393 +1,598 @@ -# -# ___ _ +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"ae44897dc32d20c421588b31a279653abf436964ad86c58885e0edbfbb2f0416","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"46d564922b082d0db93244972e8005ea6904ee5f","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55","digest":"sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55","digest":"sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55","digest":"sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19","digest":"sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} +# ___ _ _ # / _ \ | | (_) -# | |_| | __ _ ___ _ __ | |_ _ +# | |_| | __ _ ___ _ __ | |_ _ ___ # | _ |/ _` |/ _ \ '_ \| __| |/ __| -# | | | | (_| | __/ | | | |_| | ( +# | | | | (_| | __/ | | | |_| | (__ # \_| |_/\__, |\___|_| |_|\__|_|\___| # __/ | # _ _ |___/ # | | | | / _| | -# | | | | ___ _ __ _ __| |_| | _____ +# | | | | ___ _ __ _ __| |_| | _____ ____ # | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.45.0). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. # -# To update this file, edit githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87 and run: -# gh aw +# To update this file, edit githubnext/agentics/workflows/repo-status.md@main and run: +# gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# This workflow creates daily repo status reports. It gathers recent -# activity (issues, PRs, discussions, releases, code changes) and +# This workflow creates daily repo status reports. It gathers recent repository +# activity (issues, PRs, discussions, releases, code changes) and generates # engaging GitHub issues with productivity insights, community highlights, # and project recommendations. # -# Source: githubnext/agentics/workflows/daily-repo-status.md@ +# Source: githubnext/agentics/workflows/repo-status.md@main +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 # -# frontmatter-hash: +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca +# - ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 -name: "Daily Repo Status" -"on": +name: "Repo Status" +on: schedule: - - cron: "25 18 * * *" + - cron: "11 19 * * *" # Friendly format: daily (scattered) workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string permissions: {} concurrency: group: "gh-aw-${{ github.workflow }}" -run-name: "Daily Repo Status" +run-name: "Repo Status" jobs: activation: - runs-on: ubuntu- + runs-on: ubuntu-slim permissions: - contents: + actions: read + contents: read outputs: comment_id: "" comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Repo Status" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_FRONTMATTER_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - destination: /opt/gh-aw/ - - name: Check workflow file - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "daily-repo-status.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + Tools: create_issue, missing_tool, missing_data, noop + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + {{#runtime-import .github/workflows/daily-repo-status.md}} + GH_AW_PROMPT_603e0a48f86a9470_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 agent: - needs: - runs-on: ubuntu- + needs: activation + runs-on: ubuntu-latest permissions: - contents: - issues: - pull-requests: + contents: read + issues: read + pull-requests: read concurrency: group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" - GH_AW_ASSETS_MAX_SIZE_KB: - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/ - GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs. - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config. - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools. - GH_AW_WORKFLOW_ID_SANITIZED: + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_WORKFLOW_ID_SANITIZED: dailyrepostatus outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} has_patch: ${{ steps.collect_output.outputs.has_patch }} - model: ${{ steps.generate_aw_info.outputs.model }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} output_types: ${{ steps.collect_output.outputs.output_types }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - persist-credentials: - - name: Create gh-aw temp - run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir. - - name: Configure Git + persist-credentials: false + - name: Create gh-aw temp directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - # Re-authenticate git with GitHub + git config --global am.keepcr true + # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Checkout PR - id: checkout- + - name: Checkout PR branch + id: checkout-pr if: | - github.event. - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + github.event.pull_request || github.event.issue.pull_request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); await main(); - - name: Generate agentic run - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Parse integrity filter lists + id: parse-guard-vars + env: + GH_AW_BLOCKED_USERS_VAR: ${{ vars.GH_AW_GITHUB_BLOCKED_USERS || '' }} + GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} + GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - script: | - const fs = require('fs'); - - const awInfo = { - engine_id: "copilot", - engine_name: "GitHub Copilot CLI", - model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", - version: "", - agent_version: "0.0.410", - cli_version: "v0.45.0", - workflow_name: "Daily Repo Status", - experimental: false, - supports_tools_allowlist: true, - supports_http_transport: true, - run_id: context.runId, - run_number: context.runNumber, - run_attempt: process.env.GITHUB_RUN_ATTEMPT, - repository: context.repo.owner + '/' + context.repo.repo, - ref: context.ref, - sha: context.sha, - actor: context.actor, - event_name: context.eventName, - staged: false, - allowed_domains: ["defaults"], - firewall_enabled: true, - awf_version: "v0.18.0", - awmg_version: "v0.1.4", - steps: { - firewall: "squid" - }, - created_at: new Date().toISOString() - }; - - // Write to /tmp/gh-aw directory to avoid inclusion in - const tmpPath = '/tmp/gh-aw/aw_info.json'; - fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); - console.log('Generated aw_info.json at:', tmpPath); - console.log(JSON.stringify(awInfo, null, 2)); - - // Set model as output for reuse in other steps/ - core.setOutput('model', awInfo.model); - - name: Validate COPILOT_GITHUB_TOKEN - id: validate- - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot- + name: activation + path: /tmp/gh-aw + - name: Restore agent config folders from base branch + if: steps.checkout-pr.outcome == 'success' env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Install GitHub Copilot - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0. - - name: Install awf - run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18. - - name: Download container - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.18.0 ghcr.io/github/gh-aw-firewall/squid:0.18.0 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts- - - name: Write Safe Outputs + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config run: | - mkdir -p /opt/gh-aw/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/mcp-logs/ - cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"create_issue":{"max":1},"missing_data":{},"missing_tool":{},"noop":{"max":1}} - - cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' - [ + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_9b6646f0d620a78f_EOF' + {"create_issue":{"close_older_issues":true,"labels":["report","daily-status"],"max":1,"title_prefix":"[repo-status] "},"create_report_incomplete_issue":{},"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_9b6646f0d620a78f_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | { - "description": "Create a new GitHub issue for tracking bugs, feature requests, or tasks. Use this for actionable work items that need assignment, labeling, and status tracking. For reports, announcements, or status updates that don't require task tracking, use create_discussion instead. CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [report daily-status] will be automatically added.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "description_suffixes": { + "create_issue": " CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [\"report\" \"daily-status\"] will be automatically added." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "create_issue": { + "defaultMax": 1, + "fields": { "body": { - "description": "Detailed issue description in Markdown. Do NOT repeat the title as a heading since it already appears as the issue's h1. Include context, reproduction steps, or acceptance criteria as appropriate.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 }, - "labels": { - "description": "Labels to categorize the issue (e.g., 'bug', 'enhancement'). Labels must exist in the repository.", - "items": { - "type": "string" - }, + "fields": { "type": "array" }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, "parent": { - "description": "Parent issue number for creating sub-issues. This is the numeric ID from the GitHub URL (e.g., 42 in github.com/owner/repo/issues/42). Can also be a temporary_id (e.g., 'aw_abc123', 'aw_Test123') from a previously created issue in the same workflow run.", - "type": [ - "number", - "string" - ] + "issueOrPRNumber": true + }, + "repo": { + "type": "string", + "maxLength": 256 }, "temporary_id": { - "description": "Unique temporary identifier for referencing this issue before it's created. Format: 'aw_' followed by 3 to 8 alphanumeric characters (e.g., 'aw_abc1', 'aw_Test123'). Use '#aw_ID' in body text to reference other issues by their temporary_id; these are replaced with actual issue numbers after creation.", - "pattern": "^aw_[A-Za-z0-9]{4,8}$", "type": "string" }, "title": { - "description": "Concise issue title summarizing the bug, feature, or task. The title appears as the main heading, so keep it brief and descriptive.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "title", - "body" - ], - "type": "object" + } }, - "name": "create_issue" - }, - { - "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "missing_data": { + "defaultMax": 20, + "fields": { "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 }, "reason": { - "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 }, "tool": { - "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "reason" - ], - "type": "object" + } }, - "name": "missing_tool" - }, - { - "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "noop": { + "defaultMax": 1, + "fields": { "message": { - "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 } - }, - "required": [ - "message" - ], - "type": "object" + } }, - "name": "noop" - }, - { - "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" - }, - "context": { - "description": "Additional context about the missing data or where it should come from (max 256 characters).", - "type": "string" - }, - "data_type": { - "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", - "type": "string" + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "reason": { - "description": "Explanation of why this data is needed to complete the task (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 } - }, - "required": [], - "type": "object" - }, - "name": "missing_data" - } - ] - - cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' - { - "create_issue": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - }, - "labels": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": - }, - "parent": { - "issueOrPRNumber": - }, - "repo": { - "type": "string", - "maxLength": - }, - "temporary_id": { - "type": "string" - }, - "title": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": } } } - } - - - name: Generate Safe Outputs MCP Server - id: safe-outputs- + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config run: | # Generate a secure random API key (360 bits of entropy, 40+ chars) - # Mask immediately to prevent timing + # Mask immediately to prevent timing vulnerabilities API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${API_KEY}" - PORT= + PORT=3001 - # Set outputs for next + # Set outputs for next steps { echo "safe_outputs_api_key=${API_KEY}" echo "safe_outputs_port=${PORT}" @@ -395,61 +600,84 @@ jobs: echo "Safe Outputs MCP server will run on port ${PORT}" - - name: Start Safe Outputs MCP HTTP - id: safe-outputs- + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start env: DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools. - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config. - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/ + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs run: | - # Environment variables are set above to prevent template - export - export - export - export - export - export + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR - bash /opt/gh-aw/actions/start_safe_outputs_server. + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - - name: Start MCP - id: start-mcp- + - name: Start MCP Gateway + id: start-mcp-gateway env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} run: | - set -eo - mkdir -p /tmp/gh-aw/mcp- + set -eo pipefail + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" - # Export gateway environment variables for MCP config and gateway - export MCP_GATEWAY_PORT="80" + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="8080" export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${MCP_GATEWAY_API_KEY}" - export + export MCP_GATEWAY_API_KEY export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' - mkdir -p /home/runner/. - cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway. + mkdir -p /home/runner/.copilot + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_d075a7f45ab51044_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", "env": { + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "approval-labels": ${{ steps.parse-guard-vars.outputs.approval_labels }}, + "blocked-users": ${{ steps.parse-guard-vars.outputs.blocked_users }}, + "min-integrity": "none", + "repos": "all", + "trusted-users": ${{ steps.parse-guard-vars.outputs.trusted_users }} + } } }, "safeoutputs": { @@ -457,6 +685,13 @@ jobs: "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", "headers": { "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } } } }, @@ -467,212 +702,113 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - - - name: Generate workflow - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - with: - script: | - const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); - await generateWorkflowOverview(core); - - name: Create prompt with built-in + GH_AW_MCP_CONFIG_d075a7f45ab51044_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - bash /opt/gh-aw/actions/create_prompt_first. - cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" - - - cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GitHub API Access Instructions - - The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. - - - To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. - - Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). - - **IMPORTANT - temporary_id format rules:** - - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) - - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/ - - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) - - Valid alphanumeric characters: - - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) - - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, - - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto- - - Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. - - Discover available tools from the safeoutputs MCP server. - - **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. - - **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. - - - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: - {{/if}} - - - - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/workflows/daily-repo-status.md}} - - - name: Substitute - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env. - } - }); - - name: Interpolate variables and render - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); await main(); - - name: Validate prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - run: bash /opt/gh-aw/actions/validate_prompt_placeholders. - - name: Print - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - run: bash /opt/gh-aw/actions/print_prompt_summary. - - name: Clean git - run: bash /opt/gh-aw/actions/clean_git_credentials. - - name: Execute GitHub Copilot - id: + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" + - name: Execute GitHub Copilot CLI + id: agentic_execution # Copilot CLI tool arguments (sorted): - timeout-minutes: + timeout-minutes: 20 run: | - set -o - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.18.0 --skip-pull \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio. + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55,squid=sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca,agent=sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731,api-proxy=sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: - COPILOT_AGENT_RUNNER_TYPE: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config. - GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} - XDG_CONFIG_HOME: /home/ - - name: Configure Git + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - # Re-authenticate git with GitHub + git config --global am.keepcr true + # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Copy Copilot session state files to + - name: Copy Copilot session state files to logs if: always() - continue-on-error: - run: | - # Copy Copilot session state files to logs folder for artifact - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || - echo "Session state files copied successfully" - - echo "No session-state directory found at $SESSION_STATE_DIR" - - - name: Stop MCP + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" + - name: Stop MCP Gateway if: always() - continue-on-error: + continue-on-error: true env: MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | - bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" - - name: Redact secrets in + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" + - name: Redact secrets in logs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -680,343 +816,579 @@ jobs: SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload Safe + - name: Append agent step summary if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. - with: - name: safe- - path: ${{ env.GH_AW_SAFE_OUTPUTS }} - if-no-files-found: - - name: Ingest agent - id: + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_GITHUB_REFS: "" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); await main(); - - name: Upload sanitized agent - if: always() && env. - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. - with: - name: agent- - path: ${{ env.GH_AW_AGENT_OUTPUT }} - if-no-files-found: - - name: Upload engine output - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. - with: - name: - path: | - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls. - if-no-files-found: - - name: Parse agent logs for step + - name: Parse agent logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); - - name: Parse MCP Gateway logs for step + - name: Parse MCP Gateway logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); await main(); - - name: Print firewall + - name: Print firewall logs if: always() - continue-on-error: + continue-on-error: true env: - AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/ + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs run: | - # Fix permissions on firewall logs so they can be uploaded as - # AWF runs with sudo, creating files owned by - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) - if command -v awf &> /dev/null; + if command -v awf &> /dev/null; then awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" - + else echo 'AWF binary not installed, skipping firewall log summary' - - - name: Upload agent + fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary if: always() - continue-on-error: - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - name: agent- + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent path: | - /tmp/gh-aw/aw-prompts/prompt. - /tmp/gh-aw/aw_info. + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ - /tmp/gh-aw/agent-stdio. + /tmp/gh-aw/proxy-logs/ + !/tmp/gh-aw/proxy-logs/proxy-tls/ + /tmp/gh-aw/agent_usage.json + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ - if-no-files-found: + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json + if-no-files-found: ignore conclusion: needs: - - - - - - - - - if: (always()) && (needs.agent.result != 'skipped') - runs-on: ubuntu- + - activation + - agent + - detection + - safe_outputs + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') + runs-on: ubuntu-slim permissions: - contents: - issues: + contents: read + issues: write + concurrency: + group: "gh-aw-conclusion-daily-repo-status" + cancel-in-progress: false + queue: max outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/safeoutputs/ - - name: Setup agent output environment + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f - - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process No-Op - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/noop.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); await main(); - - name: Record Missing - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); await main(); - - name: Handle Agent - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "daily-repo-status" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} - GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + - name: Record missing tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); await main(); - - name: Handle No-Op - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); + await main(); + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" + GH_AW_WORKFLOW_ID: "daily-repo-status" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "20" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); await main(); detection: needs: - if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' - runs-on: ubuntu- - permissions: {} - concurrency: - group: "gh-aw-copilot-${{ github.workflow }}" - timeout-minutes: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') + runs-on: ubuntu-latest + permissions: + contents: read outputs: - success: ${{ steps.parse_results.outputs.success }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/threat-detection/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - name: agent- - path: /tmp/gh-aw/threat-detection/ - - name: Echo agent output + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca + - name: Check if detection needed + id: detection_guard + if: always() env: - AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - echo "Agent output-types: $AGENT_OUTPUT_TYPES" - - name: Setup threat - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - WORKFLOW_NAME: "Daily Repo Status" + WORKFLOW_NAME: "Repo Status" WORKFLOW_DESCRIPTION: "This workflow creates daily repo status reports. It gathers recent repository\nactivity (issues, PRs, discussions, releases, code changes) and generates\nengaging GitHub issues with productivity insights, community highlights,\nand project recommendations." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); await main(); - - name: Ensure threat-detection directory and + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - mkdir -p /tmp/gh-aw/threat- - touch /tmp/gh-aw/threat-detection/detection. - - name: Validate COPILOT_GITHUB_TOKEN - id: validate- - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot- + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Install GitHub Copilot - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0. - - name: Execute GitHub Copilot - id: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool shell(cat) - # --allow-tool shell(grep) - # --allow-tool shell(head) - # --allow-tool shell(jq) - # --allow-tool shell(ls) - # --allow-tool shell(tail) - # --allow-tool shell(wc) - timeout-minutes: + timeout-minutes: 20 run: | - set -o - COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" - mkdir -p /tmp/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/agent/ - mkdir -p /tmp/gh-aw/sandbox/agent/logs/ - copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection. + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: - COPILOT_AGENT_RUNNER_TYPE: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} - XDG_CONFIG_HOME: /home/ - - name: Parse threat detection - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - - name: Upload threat detection + name: detection + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" with: - name: threat-detection. - path: /tmp/gh-aw/threat-detection/detection. - if-no-files-found: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } safe_outputs: needs: - - - - - if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') - runs-on: ubuntu- + - activation + - agent + - detection + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' + runs-on: ubuntu-slim permissions: - contents: - issues: - timeout-minutes: + contents: read + issues: write + timeout-minutes: 15 env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/daily-repo-status" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" GH_AW_WORKFLOW_ID: "daily-repo-status" - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + created_issue_number: ${{ steps.process_safe_outputs.outputs.created_issue_number }} + created_issue_url: ${{ steps.process_safe_outputs.outputs.created_issue_url }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/safeoutputs/ - - name: Setup agent output environment + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f - - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process Safe - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"create_report_incomplete_issue\":{},\"mentions\":{\"enabled\":false},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore diff --git a/.github/workflows/daily-repo-status.md b/.github/workflows/daily-repo-status.md index 431b4afb91a6..8debf5708437 100644 --- a/.github/workflows/daily-repo-status.md +++ b/.github/workflows/daily-repo-status.md @@ -22,15 +22,19 @@ tools: # reading issues, pull requests and comments from 3rd-parties # If in a private repo this has no particular effect. lockdown: false + min-integrity: none # This workflow is allowed to examine and comment on any issues safe-outputs: + mentions: false + allowed-github-references: [] create-issue: title-prefix: "[repo-status] " labels: [report, daily-status] -source: githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87 + close-older-issues: true +source: githubnext/agentics/workflows/repo-status.md@main --- -# Daily Repo Status +# Repo Status Create an upbeat daily status report for the repo as a GitHub issue. diff --git a/.github/workflows/docker-cloudstack-simulator.yml b/.github/workflows/docker-cloudstack-simulator.yml index 8d23ac449dd5..dc00968563d0 100644 --- a/.github/workflows/docker-cloudstack-simulator.yml +++ b/.github/workflows/docker-cloudstack-simulator.yml @@ -35,10 +35,10 @@ concurrency: jobs: build: if: github.repository == 'apache/cloudstack' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Login to Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKERHUB_USER }} @@ -47,7 +47,9 @@ jobs: - name: Set Docker repository name run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set ACS version run: echo "ACS_VERSION=$(grep '' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 2410f7b9e457..94abbb0908cc 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -1,4 +1,5 @@ -# +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"d4b8988df7c60cd416200769cc6bc7f1aab5bb3128df0b9a83a35e061b4da111","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"46d564922b082d0db93244972e8005ea6904ee5f","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55","digest":"sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55","digest":"sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55","digest":"sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19","digest":"sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) # | |_| | __ _ ___ _ __ | |_ _ ___ @@ -13,28 +14,57 @@ # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.45.0). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. # -# To update this file, edit github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 and run: +# To update this file, edit github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 and run: # gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # # -# Source: github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +# Source: github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 # # Resolved workflow manifest: # Imports: -# - github/gh-aw/.github/workflows/shared/reporting.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +# - github/gh-aw/.github/workflows/shared/noop-reminder.md@359795d49ada21681ab616bd4cbcb144a7387115 +# - github/gh-aw/.github/workflows/shared/reporting.md@359795d49ada21681ab616bd4cbcb144a7387115 +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 # -# frontmatter-hash: 7bc83974fa1e47c12b40c3333872e4126711d5c6624022cc78b76047289d8b63 +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca +# - ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 name: "Issue Triage Agent" -"on": +on: schedule: - - cron: "0 14 * * 1-5" + - cron: "49 14 * * 1-5" + # Friendly format: daily around 14:00 on weekdays (scattered) workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string permissions: {} @@ -47,324 +77,503 @@ jobs: activation: runs-on: ubuntu-slim permissions: + actions: read contents: read outputs: comment_id: "" comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions - - name: Check workflow file timestamps - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_FRONTMATTER_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_FRONTMATTER_EMOJI: "🔧" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "issue-triage-agent.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_e560c36b9148ef78_EOF' + + GH_AW_PROMPT_e560c36b9148ef78_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_e560c36b9148ef78_EOF' + + Tools: add_comment, add_labels, missing_tool, missing_data, noop + + GH_AW_PROMPT_e560c36b9148ef78_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_e560c36b9148ef78_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_e560c36b9148ef78_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_e560c36b9148ef78_EOF' + + {{#runtime-import .github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md}} + {{#runtime-import .github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_noop-reminder.md}} + {{#runtime-import .github/workflows/issue-triage-agent.md}} + GH_AW_PROMPT_e560c36b9148ef78_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 agent: needs: activation runs-on: ubuntu-latest permissions: - contents: read - issues: read + issues: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json GH_AW_WORKFLOW_ID_SANITIZED: issuetriageagent outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} has_patch: ${{ steps.collect_output.outputs.has_patch }} - model: ${{ steps.generate_aw_info.outputs.model }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} output_types: ${{ steps.collect_output.outputs.output_types }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Create gh-aw temp directory - run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git config --global am.keepcr true # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Generate agentic run info - id: generate_aw_info - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const fs = require('fs'); - - const awInfo = { - engine_id: "copilot", - engine_name: "GitHub Copilot CLI", - model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", - version: "", - agent_version: "0.0.410", - cli_version: "v0.45.0", - workflow_name: "Issue Triage Agent", - experimental: false, - supports_tools_allowlist: true, - supports_http_transport: true, - run_id: context.runId, - run_number: context.runNumber, - run_attempt: process.env.GITHUB_RUN_ATTEMPT, - repository: context.repo.owner + '/' + context.repo.repo, - ref: context.ref, - sha: context.sha, - actor: context.actor, - event_name: context.eventName, - staged: false, - allowed_domains: ["defaults"], - firewall_enabled: true, - awf_version: "v0.18.0", - awmg_version: "v0.1.4", - steps: { - firewall: "squid" - }, - created_at: new Date().toISOString() - }; - - // Write to /tmp/gh-aw directory to avoid inclusion in PR - const tmpPath = '/tmp/gh-aw/aw_info.json'; - fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); - console.log('Generated aw_info.json at:', tmpPath); - console.log(JSON.stringify(awInfo, null, 2)); - - // Set model as output for reuse in other steps/jobs - core.setOutput('model', awInfo.model); - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - name: Install GitHub Copilot CLI - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.410 - - name: Install awf binary - run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18.0 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 - name: Determine automatic lockdown mode for GitHub MCP Server id: determine-automatic-lockdown - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) env: GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} with: script: | - const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); await determineAutomaticLockdown(github, context, core); + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.18.0 ghcr.io/github/gh-aw-firewall/squid:0.18.0 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine - - name: Write Safe Outputs Config + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config run: | - mkdir -p /opt/gh-aw/safeoutputs + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"add_comment":{"max":1},"add_labels":{"allowed":["bug","feature","enhancement","documentation","question","help-wanted","good-first-issue"],"max":3},"missing_data":{},"missing_tool":{},"noop":{"max":1}} - GH_AW_SAFE_OUTPUTS_CONFIG_EOF - cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' - [ + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_c8eaf0ada4607ff7_EOF' + {"add_comment":{"max":1},"add_labels":{"allowed":["bug","feature","enhancement","documentation","question","help-wanted","good-first-issue"]},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_c8eaf0ada4607ff7_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | + { + "description_suffixes": { + "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added. Supports reply_to_id for discussion threading.", + "add_labels": " CONSTRAINTS: Only these labels are allowed: [\"bug\" \"feature\" \"enhancement\" \"documentation\" \"question\" \"help-wanted\" \"good-first-issue\"]." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | { - "description": "Add a comment to an existing GitHub issue, pull request, or discussion. Use this to provide feedback, answer questions, or add information to an existing conversation. For creating new items, use create_issue, create_discussion, or create_pull_request instead. IMPORTANT: Comments are subject to validation constraints enforced by the MCP server - maximum 65536 characters for the complete comment (including footer which is added automatically), 10 mentions (@username), and 50 links. Exceeding these limits will result in an immediate error with specific guidance. CONSTRAINTS: Maximum 1 comment(s) can be added.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "add_comment": { + "defaultMax": 1, + "fields": { "body": { - "description": "The comment text in Markdown format. This is the 'body' field - do not use 'comment_body' or other variations. Provide helpful, relevant information that adds value to the conversation. CONSTRAINTS: The complete comment (your body text + automatically added footer) must not exceed 65536 characters total. Maximum 10 mentions (@username), maximum 50 links (http/https URLs). A footer (~200-500 characters) is automatically appended with workflow attribution, so leave adequate space. If these limits are exceeded, the tool call will fail with a detailed error message indicating which constraint was violated.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "item_number": { - "description": "The issue, pull request, or discussion number to comment on. This is the numeric ID from the GitHub URL (e.g., 123 in github.com/owner/repo/issues/123). If omitted, the tool will attempt to resolve the target from the current workflow context (triggering issue, PR, or discussion).", - "type": "number" + "issueOrPRNumber": true + }, + "reply_to_id": { + "type": "string", + "maxLength": 256 + }, + "repo": { + "type": "string", + "maxLength": 256 } - }, - "required": [ - "body" - ], - "type": "object" + } }, - "name": "add_comment" - }, - { - "description": "Add labels to an existing GitHub issue or pull request for categorization and filtering. Labels must already exist in the repository. For creating new issues with labels, use create_issue with the labels property instead. CONSTRAINTS: Only these labels are allowed: [bug feature enhancement documentation question help-wanted good-first-issue].", - "inputSchema": { - "additionalProperties": false, - "properties": { + "add_labels": { + "defaultMax": 5, + "fields": { "item_number": { - "description": "Issue or PR number to add labels to. This is the numeric ID from the GitHub URL (e.g., 456 in github.com/owner/repo/issues/456). If omitted, adds labels to the item that triggered this workflow.", - "type": "number" + "issueNumberOrTemporaryId": true }, "labels": { - "description": "Label names to add (e.g., ['bug', 'priority-high']). Labels must exist in the repository.", - "items": { - "type": "string" - }, - "type": "array" + "required": true, + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "repo": { + "type": "string", + "maxLength": 256 } - }, - "type": "object" + } }, - "name": "add_labels" - }, - { - "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 512 }, "reason": { - "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 }, "tool": { - "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "reason" - ], - "type": "object" + } }, - "name": "missing_tool" - }, - { - "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "noop": { + "defaultMax": 1, + "fields": { "message": { - "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 } - }, - "required": [ - "message" - ], - "type": "object" + } }, - "name": "noop" - }, - { - "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" - }, - "context": { - "description": "Additional context about the missing data or where it should come from (max 256 characters).", - "type": "string" - }, - "data_type": { - "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", - "type": "string" + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "reason": { - "description": "Explanation of why this data is needed to complete the task (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 } - }, - "required": [], - "type": "object" - }, - "name": "missing_data" - } - ] - GH_AW_SAFE_OUTPUTS_TOOLS_EOF - cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' - { - "add_comment": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "item_number": { - "issueOrPRNumber": true - } - } - }, - "add_labels": { - "defaultMax": 5, - "fields": { - "item_number": { - "issueOrPRNumber": true - }, - "labels": { - "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": 512 - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 } } } - } - GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); - name: Generate Safe Outputs MCP Server Config id: safe-outputs-config run: | @@ -387,59 +596,81 @@ jobs: id: safe-outputs-start env: DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs run: | # Environment variables are set above to prevent template injection export DEBUG + export GH_AW_SAFE_OUTPUTS export GH_AW_SAFE_OUTPUTS_PORT export GH_AW_SAFE_OUTPUTS_API_KEY export GH_AW_SAFE_OUTPUTS_TOOLS_PATH export GH_AW_SAFE_OUTPUTS_CONFIG_PATH export GH_AW_MCP_LOG_DIR - bash /opt/gh-aw/actions/start_safe_outputs_server.sh + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - name: Start MCP Gateway id: start-mcp-gateway env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} - GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} + GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} run: | set -eo pipefail - mkdir -p /tmp/gh-aw/mcp-config + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" # Export gateway environment variables for MCP config and gateway script - export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_PORT="8080" export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${MCP_GATEWAY_API_KEY}" export MCP_GATEWAY_API_KEY export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + export GH_AW_MCP_CLI_SERVERS='["safeoutputs"]' + echo GH_AW_MCP_CLI_SERVERS='["safeoutputs"]' >> "$GITHUB_ENV" + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_88b8311e90d25032_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", "env": { - "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "issues,labels" + }, + "guard-policies": { + "allow-only": { + "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", + "repos": "$GITHUB_MCP_GUARD_REPOS" + } } }, "safeoutputs": { @@ -447,6 +678,13 @@ jobs: "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", "headers": { "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } } } }, @@ -457,199 +695,96 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_EOF - - name: Generate workflow overview - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); - await generateWorkflowOverview(core); - - name: Create prompt with built-in context - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - bash /opt/gh-aw/actions/create_prompt_first.sh - cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" - - GH_AW_PROMPT_EOF - cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GitHub API Access Instructions - - The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. - - - To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. - - Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). - - **IMPORTANT - temporary_id format rules:** - - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) - - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i - - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) - - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 - - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) - - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 - - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate - - Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. - - Discover available tools from the safeoutputs MCP server. - - **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. - - **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. - - - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: __GH_AW_GITHUB_ACTOR__ - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: __GH_AW_GITHUB_REPOSITORY__ - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: __GH_AW_GITHUB_WORKSPACE__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ - {{/if}} - - - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/aw/imports/github/gh-aw/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github_workflows_shared_reporting.md}} - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/workflows/issue-triage-agent.md}} - GH_AW_PROMPT_EOF - - name: Substitute placeholders - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution function - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE - } - }); - - name: Interpolate variables and render templates - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + GH_AW_MCP_CONFIG_88b8311e90d25032_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); await main(); - - name: Validate prompt placeholders - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh - - name: Print prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - run: bash /opt/gh-aw/actions/print_prompt_summary.sh - - name: Clean git credentials - run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): timeout-minutes: 5 run: | set -o pipefail - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.18.0 --skip-pull \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55,squid=sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca,agent=sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731,api-proxy=sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: + AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json - GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git config --global am.keepcr true # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - name: Copy Copilot session state files to logs if: always() continue-on-error: true - run: | - # Copy Copilot session state files to logs folder for artifact collection - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; then - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true - echo "Session state files copied successfully" - else - echo "No session-state directory found at $SESSION_STATE_DIR" - fi + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" - name: Stop MCP Gateway if: always() continue-on-error: true @@ -658,15 +793,15 @@ jobs: MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | - bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" - name: Redact secrets in logs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -674,62 +809,51 @@ jobs: SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload Safe Outputs + - name: Append agent step summary if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: safe-output - path: ${{ env.GH_AW_SAFE_OUTPUTS }} - if-no-files-found: warn + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs + if: always() + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true - name: Ingest agent output id: collect_output if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); await main(); - - name: Upload sanitized agent output - if: always() && env.GH_AW_AGENT_OUTPUT - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: agent-output - path: ${{ env.GH_AW_AGENT_OUTPUT }} - if-no-files-found: warn - - name: Upload engine output files - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: agent_outputs - path: | - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls.log - if-no-files-found: ignore - name: Parse agent logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); - name: Parse MCP Gateway logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); await main(); - name: Print firewall logs if: always() @@ -737,28 +861,65 @@ jobs: env: AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs run: | - # Fix permissions on firewall logs so they can be uploaded as artifacts + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts # AWF runs with sudo, creating files owned by root - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) if command -v awf &> /dev/null; then awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" else echo 'AWF binary not installed, skipping firewall log summary' fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi - name: Upload agent artifacts if: always() continue-on-error: true - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: agent-artifacts + name: agent path: | /tmp/gh-aw/aw-prompts/prompt.txt - /tmp/gh-aw/aw_info.json + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent_usage.json /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json if-no-files-found: ignore conclusion: @@ -767,207 +928,370 @@ jobs: - agent - detection - safe_outputs - if: (always()) && (needs.agent.result != 'skipped') + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') runs-on: ubuntu-slim permissions: contents: read discussions: write issues: write pull-requests: write + concurrency: + group: "gh-aw-conclusion-issue-triage-agent" + cancel-in-progress: false + queue: max outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/safeoutputs/ + name: agent + path: /tmp/gh-aw/ - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process No-Op Messages + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages id: noop - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: 1 + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/noop.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); await main(); - - name: Record Missing Tool + - name: Record missing tool id: missing_tool - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); await main(); - - name: Handle Agent Failure - id: handle_agent_failure - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "issue-triage-agent" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); await main(); - - name: Handle No-Op Message - id: handle_noop_message - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" + GH_AW_WORKFLOW_ID: "issue-triage-agent" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "5" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); await main(); detection: - needs: agent - if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + needs: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') runs-on: ubuntu-latest - permissions: {} - timeout-minutes: 10 + permissions: + contents: read outputs: - success: ${{ steps.parse_results.outputs.success }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 - with: - destination: /opt/gh-aw/actions - - name: Download agent artifacts - continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - name: agent-artifacts - path: /tmp/gh-aw/threat-detection/ + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - name: agent-output - path: /tmp/gh-aw/threat-detection/ - - name: Echo agent output types + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca + - name: Check if detection needed + id: detection_guard + if: always() env: - AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} run: | - echo "Agent output-types: $AGENT_OUTPUT_TYPES" + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true - name: Setup threat detection - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Issue Triage Agent" WORKFLOW_DESCRIPTION: "No description provided" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); await main(); - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | mkdir -p /tmp/gh-aw/threat-detection touch /tmp/gh-aw/threat-detection/detection.log - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false - name: Install GitHub Copilot CLI - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.410 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 - name: Execute GitHub Copilot CLI - id: agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool shell(cat) - # --allow-tool shell(grep) - # --allow-tool shell(head) - # --allow-tool shell(jq) - # --allow-tool shell(ls) - # --allow-tool shell(tail) - # --allow-tool shell(wc) timeout-minutes: 20 run: | set -o pipefail - COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" - mkdir -p /tmp/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/agent/ - mkdir -p /tmp/gh-aw/sandbox/agent/logs/ - copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: + AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] XDG_CONFIG_HOME: /home/runner - - name: Parse threat detection results - id: parse_results - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - name: Upload threat detection log - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: threat-detection.log + name: detection path: /tmp/gh-aw/threat-detection/detection.log if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + with: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } safe_outputs: needs: + - activation - agent - detection - if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' runs-on: ubuntu-slim permissions: contents: read @@ -976,41 +1300,89 @@ jobs: pull-requests: write timeout-minutes: 15 env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/issue-triage-agent" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} + GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" + GH_AW_WORKFLOW_EMOJI: "🔧" GH_AW_WORKFLOW_ID: "issue-triage-agent" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + comment_id: ${{ steps.process_safe_outputs.outputs.comment_id }} + comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/safeoutputs/ + name: agent + path: /tmp/gh-aw/ - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" - name: Process Safe Outputs id: process_safe_outputs - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore diff --git a/.github/workflows/issue-triage-agent.md b/.github/workflows/issue-triage-agent.md index 06ff227b740d..2d382f9117c4 100644 --- a/.github/workflows/issue-triage-agent.md +++ b/.github/workflows/issue-triage-agent.md @@ -1,11 +1,12 @@ --- on: - schedule: 0 14 * * 1-5 + schedule: daily around 14:00 on weekdays workflow_dispatch: null permissions: issues: read imports: -- github/gh-aw/.github/workflows/shared/reporting.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +- github/gh-aw/.github/workflows/shared/reporting.md@359795d49ada21681ab616bd4cbcb144a7387115 +- github/gh-aw/.github/workflows/shared/noop-reminder.md@359795d49ada21681ab616bd4cbcb144a7387115 safe-outputs: add-comment: {} add-labels: @@ -17,10 +18,12 @@ safe-outputs: - question - help-wanted - good-first-issue -source: github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +emoji: 🔧 +source: github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 strict: true timeout-minutes: 5 tools: + cli-proxy: true github: toolsets: - issues @@ -45,7 +48,7 @@ Hi @{author}! I've categorized this issue as **{label_name}** based on the follo **Reasoning**: {brief_explanation_of_why_this_label}
-View Triage Details +View Triage Details #### Analysis - **Keywords detected**: {list_of_keywords_that_matched} @@ -76,3 +79,13 @@ For efficiency, if multiple issues are triaged in a single run: 3. Optionally: Create a discussion summarizing all triage actions for that run This provides both per-issue context and batch visibility. + +## Labels + +- `bug`: Indicates a problem or error in the code that needs fixing. +- `feature`: Represents a new feature request or enhancement to existing functionality. +- `enhancement`: Suggests improvements to existing features or code. +- `documentation`: Pertains to issues related to documentation, such as missing or unclear docs. +- `question`: Used for issues that are asking for clarification or have questions about the project. +- `help-wanted`: Indicates that the issue is a good candidate for external contributions and help +- `good-first-issue`: Marks issues that are suitable for newcomers to the project, often with simpler scope. diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 7ccd6600ab97..3d510fa5dc34 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -15,54 +15,51 @@ # specific language governing permissions and limitations # under the License. -name: Main Branch Sonar Quality Check - +name: Sonar Quality Check (Main) +permissions: + contents: read on: push: branches: - main - -permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: if: github.repository == 'apache/cloudstack' - name: Main Sonar JaCoCo Build - runs-on: ubuntu-22.04 + name: Sonar JaCoCo Coverage + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - - - name: Run Tests with Coverage + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss + - name: Run Build and Tests with Coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack + run: mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.branch.name=${{ github.ref_name }} + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/merge-conflict-checker.yml b/.github/workflows/merge-conflict-checker.yml index a997cb94ccc0..f23719f7183a 100644 --- a/.github/workflows/merge-conflict-checker.yml +++ b/.github/workflows/merge-conflict-checker.yml @@ -17,28 +17,26 @@ name: "PR Merge Conflict Check" on: - push: - pull_request: - types: [opened, synchronize, reopened] + schedule: + - cron: '*/10 * * * *' + workflow_dispatch: -permissions: # added using https://github.com/step-security/secure-workflows - contents: read +permissions: {} concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: "gh-aw-${{ github.workflow }}" jobs: triage: permissions: - pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs - runs-on: ubuntu-22.04 + pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs + runs-on: ubuntu-24.04 steps: - - name: Conflict Check - uses: eps1lon/actions-label-merge-conflict@v2.0.0 - with: - repoToken: "${{ secrets.GITHUB_TOKEN }}" - dirtyLabel: "status:has-conflicts" - removeOnDirtyLabel: "status:ready-for-review" - continueOnMissingPermissions: true - commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." + - name: Conflict Check + uses: eps1lon/actions-label-merge-conflict@1df065ebe6e3310545d4f4c4e862e43bdca146f0 # v3.0.3 + with: + repoToken: "${{ secrets.GITHUB_TOKEN }}" + dirtyLabel: "status:has-conflicts" + removeOnDirtyLabel: "status:ready-for-review" + continueOnMissingPermissions: true + commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 895a597659de..1ea00be9112f 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -29,17 +29,23 @@ concurrency: jobs: pre-commit: name: Run pre-commit - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Out - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + cache: 'pip' - name: Install - run: | - python -m pip install --upgrade pip - pip install pre-commit + run: pip install pre-commit - name: Set PY run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v5 + - name: Cache pre-commit environments + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index 21b8e197d825..a36a956856e7 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -16,32 +16,27 @@ # under the License. name: License Check - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - java-version: '17' - distribution: 'adopt' - architecture: x64 - cache: maven + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: RAT licence checks run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check - name: Rat Report if: always() diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 9f5c3a194bc7..e614d1864824 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -16,58 +16,52 @@ # under the License. name: Sonar Quality Check - -on: [pull_request] - permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - + contents: read + pull-requests: write +on: + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: - if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository name: Sonar JaCoCo Coverage - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: "refs/pull/${{ github.event.number }}/merge" fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Run Build and Tests with Coverage - id: coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube + if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} PR_ID: ${{ github.event.pull_request.number }} HEADREF: ${{ github.event.pull_request.head.ref }} run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 842e4497a4ad..c0da5f98cc8e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: stale-issue-message: 'This issue is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' stale-pr-message: 'This PR is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' @@ -41,7 +41,7 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' - - uses: actions/stale@v10 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: stale-issue-label: 'archive' days-before-stale: 240 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 4580b6bbd5da..0a6cd5f3ea5f 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -28,15 +28,19 @@ permissions: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Node - uses: actions/setup-node@v5 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 16 + cache: 'npm' + cache-dependency-path: 'ui/package-lock.json' - name: Env details run: | @@ -55,7 +59,7 @@ jobs: npm run lint npm run test:unit - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 if: github.repository == 'apache/cloudstack' with: working-directory: ui diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 755ae125edf0..91537e25267e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -151,7 +151,7 @@ repos: ^server/src/test/resources/certs/rsa_self_signed\.key$| ^services/console-proxy/rdpconsole/src/test/doc/rdp-key\.pem$| ^systemvm/agent/certs/localhost\.key$| - ^systemvm/agent/certs/realhostip\.key$| + ^systemvm/agent/certs/systemvm\.key$| ^test/integration/smoke/test_ssl_offloading\.py$ - id: end-of-file-fixer exclude: \.vhd$|\.svg$ diff --git a/README.md b/README.md index a5aacb49f6b5..852674ab7b1f 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Sec The following provides more details on the included cryptographic software: * CloudStack makes use of JaSypt cryptographic libraries. -* CloudStack has a system requirement of MySQL, and uses native database encryption functionality. +* CloudStack requires a MySQL-compatible database (MariaDB or MySQL), and uses native database encryption functionality. * CloudStack makes use of the Bouncy Castle general-purpose encryption library. * CloudStack can optionally interact with and control OpenSwan-based VPNs. * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation. diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index ba4a3874664a..0a459d8d4fc5 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -78,6 +78,14 @@ zone=default # Generated with "uuidgen". local.storage.uuid= +# Enable TLS for image server transfers. The keys are read from: +# cert file = /etc/cloudstack/agent/cloud.crt +# key file = /etc/cloudstack/agent/cloud.key +image.server.tls.enabled=true + +# The Address for the network interface that the image server listens on. If not specified, it will listen on the Management network. +#image.server.listen.address= + # Location for KVM virtual router scripts. # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/". domr.scripts.dir=scripts/network/domr/kvm diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index e69a7efdc9c7..18a6b6df1006 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -123,6 +123,20 @@ public class AgentProperties{ */ public static final Property LOCAL_STORAGE_PATH = new Property<>("local.storage.path", "/var/lib/libvirt/images/"); + /** + * Enables TLS on the KVM image server transfer endpoint.
+ * Data type: Boolean.
+ * Default value: true + */ + public static final Property IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", true); + + /** + * The IP address that the KVM image server listens on.
+ * Data type: String.
+ * Default value: null + */ + public static final Property IMAGE_SERVER_LISTEN_ADDRESS = new Property<>("image.server.listen.address", null, String.class); + /** * Directory where Qemu sockets are placed.
* These sockets are for the Qemu Guest Agent and SSVM provisioning.
diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index e26cc1e9f029..9af6c731fd24 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -51,6 +51,7 @@ public class VirtualMachineTO { private long minRam; private long maxRam; + private long requestedRam; private String hostName; private String arch; private String os; @@ -207,15 +208,20 @@ public long getMinRam() { return minRam; } - public void setRam(long minRam, long maxRam) { + public void setRam(long minRam, long maxRam, long requestedRam) { this.minRam = minRam; this.maxRam = maxRam; + this.requestedRam = requestedRam; } public long getMaxRam() { return maxRam; } + public long getRequestedRam() { + return requestedRam; + } + public String getHostName() { return hostName; } diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa290..5d028d31d5b6 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 8f7e773070f0..22d796d4a775 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing; + random, firstfit, userdispersing, firstfitleastconsumed; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/com/cloud/ha/Investigator.java b/api/src/main/java/com/cloud/ha/Investigator.java index 88d802a1ce44..00371d395f5a 100644 --- a/api/src/main/java/com/cloud/ha/Investigator.java +++ b/api/src/main/java/com/cloud/ha/Investigator.java @@ -26,17 +26,19 @@ public interface Investigator extends Adapter { * Returns if the vm is still alive. * * @param vm to work on. + * @return true if vm is alive, otherwise false */ - public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM; + boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM; - public Status isAgentAlive(Host agent); + /** + * Returns the agent status of the host. + * + * @param host + * @return status of the host agent + */ + Status getHostAgentStatus(Host host); class UnknownVM extends Exception { - - /** - * - */ private static final long serialVersionUID = 1L; - }; } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index b52348201516..8b14cfd3a390 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -63,6 +63,8 @@ public static String[] toStrings(Host.Type... types) { String HOST_OVFTOOL_VERSION = "host.ovftool.version"; String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; String HOST_SSH_PORT = "host.ssh.port"; + String GUEST_OS_CATEGORY_ID = "guest.os.category.id"; + String GUEST_OS_RULE = "guest.os.rule"; int DEFAULT_SSH_PORT = 22; diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 0846306f70f9..e41eb880ffd5 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -510,4 +510,6 @@ public void setIp6Address(String ip6Address) { Integer getPrivateMtu(); Integer getNetworkCidrSize(); + + boolean getKeepMacAddressOnPublicNic(); } diff --git a/api/src/main/java/com/cloud/network/NetworkProfile.java b/api/src/main/java/com/cloud/network/NetworkProfile.java index 2e8efb489308..d690344a0e38 100644 --- a/api/src/main/java/com/cloud/network/NetworkProfile.java +++ b/api/src/main/java/com/cloud/network/NetworkProfile.java @@ -385,6 +385,11 @@ public Integer getNetworkCidrSize() { return networkCidrSize; } + @Override + public boolean getKeepMacAddressOnPublicNic() { + return true; + } + @Override public String toString() { return String.format("NetworkProfile %s", diff --git a/api/src/main/java/com/cloud/network/vpc/Vpc.java b/api/src/main/java/com/cloud/network/vpc/Vpc.java index b94089d2d433..a0686e2bf7d0 100644 --- a/api/src/main/java/com/cloud/network/vpc/Vpc.java +++ b/api/src/main/java/com/cloud/network/vpc/Vpc.java @@ -107,4 +107,6 @@ public enum State { String getIp6Dns2(); boolean useRouterIpAsResolver(); + + boolean getKeepMacAddressOnPublicNic(); } diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java index c1546609d2b7..3d0ba43263f5 100644 --- a/api/src/main/java/com/cloud/network/vpc/VpcService.java +++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java @@ -58,7 +58,7 @@ public interface VpcService { */ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize, - Long asNumber, List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException; + Long asNumber, List bgpPeerIds, Boolean useVrIpResolver, boolean keepMacAddressOnPublicNic) throws ResourceAllocationException; /** * Persists VPC record in the database @@ -104,7 +104,7 @@ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, Strin * @throws ResourceUnavailableException if during restart some resources may not be available * @throws InsufficientCapacityException if for instance no address space, compute or storage is sufficiently available */ - Vpc updateVpc(long vpcId, String vpcName, String displayText, String customId, Boolean displayVpc, Integer mtu, String sourceNatIp) throws ResourceUnavailableException, InsufficientCapacityException; + Vpc updateVpc(long vpcId, String vpcName, String displayText, String customId, Boolean displayVpc, Integer mtu, String sourceNatIp, Boolean keepMacAddressOnPublicNic) throws ResourceUnavailableException, InsufficientCapacityException; /** * Lists VPC(s) based on the parameters passed to the API call diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 1a9bcc6ee98b..43ead8d287b9 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.cloud.dc.DataCenter; import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.user.Account; @@ -70,6 +71,10 @@ public interface VolumeApiService { */ Volume allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException; + Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, String name, + Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId) + throws ResourceAllocationException; + /** * Creates the volume based on the given criteria * @@ -80,6 +85,8 @@ public interface VolumeApiService { */ Volume createVolume(CreateVolumeCmd cmd); + Volume createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display); + /** * Resizes the volume based on the given criteria * @@ -203,4 +210,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId, Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); + + Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone, boolean encryptEnabledOnly); } diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 4145e2b89eb3..fc450e9179c5 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -88,10 +88,16 @@ User createUser(String userName, String password, String firstName, String lastN Account getActiveAccountById(long accountId); + Account getActiveAccountByUuid(String accountUuid); + Account getAccount(long accountId); + Account getAccountByUuid(String accountUuid); + User getActiveUser(long userId); + User getOneActiveUserForAccount(Account account); + User getUserIncludingRemoved(long userId); boolean isRootAdmin(Long accountId); diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 9e56bf4f17b2..33cc6da70812 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -130,4 +130,10 @@ public interface VmDetailConstants { String EXTERNAL_DETAIL_PREFIX = "External:"; String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details"; String CLOUDSTACK_VLAN = "cloudstack.vlan"; + + // KVM Checkpoints related + String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id"; + String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time"; + String LAST_CHECKPOINT_ID = "last.checkpoint.id"; + String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 6fb7cb0612c0..17416f08690a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -69,6 +69,8 @@ public class ApiConstants { public static final String BACKUP_VM_OFFERING_REMOVED = "vmbackupofferingremoved"; public static final String IS_BACKUP_VM_EXPUNGED = "isbackupvmexpunged"; public static final String BACKUP_TOTAL = "backuptotal"; + public static final String BALANCE = "balance"; + public static final String BALANCES = "balances"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; @@ -77,6 +79,7 @@ public class ApiConstants { public static final String BOOTABLE = "bootable"; public static final String BIND_DN = "binddn"; public static final String BIND_PASSWORD = "bindpass"; + public static final String BLANK_INSTANCE = "blankinstance"; public static final String BUS_ADDRESS = "busaddress"; public static final String BYTES_READ_RATE = "bytesreadrate"; public static final String BYTES_READ_RATE_MAX = "bytesreadratemax"; @@ -171,6 +174,7 @@ public class ApiConstants { public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISKS_DETAILS = "datadisksdetails"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; + public static final String DATE = "date"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DELETE_PROTECTION = "deleteprotection"; public static final String DESCRIPTION = "description"; @@ -217,6 +221,7 @@ public class ApiConstants { public static final String DOMAIN_PATH = "domainpath"; public static final String DOMAIN_ID = "domainid"; public static final String DOMAIN__ID = "domainId"; + public static final String DUMMY = "dummy"; public static final String DURATION = "duration"; public static final String ELIGIBLE = "eligible"; public static final String EMAIL = "email"; @@ -260,6 +265,7 @@ public class ApiConstants { public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; public static final String FOR_PROVIDER = "forprovider"; + public static final String FROM_CHECKPOINT_ID = "fromcheckpointid"; public static final String FULL_PATH = "fullpath"; public static final String GATEWAY = "gateway"; public static final String IP6_GATEWAY = "ip6gateway"; @@ -332,6 +338,7 @@ public class ApiConstants { public static final String IS_2FA_VERIFIED = "is2faverified"; public static final String IS_2FA_MANDATED = "is2famandated"; + public static final String IS_ACTIVE = "isactive"; public static final String IS_ASYNC = "isasync"; public static final String IP_AVAILABLE = "ipavailable"; public static final String IP_LIMIT = "iplimit"; @@ -442,6 +449,7 @@ public class ApiConstants { public static final String MAX_VGPU_PER_PHYSICAL_GPU = "maxvgpuperphysicalgpu"; public static final String GUEST_OS_LIST = "guestoslist"; public static final String GUEST_OS_COUNT = "guestoscount"; + public static final String GUEST_OS_RULE = "guestosrule"; public static final String OS_MAPPING_CHECK_ENABLED = "osmappingcheckenabled"; public static final String OUTOFBANDMANAGEMENT_POWERSTATE = "outofbandmanagementpowerstate"; public static final String OUTOFBANDMANAGEMENT_ENABLED = "outofbandmanagementenabled"; @@ -595,6 +603,8 @@ public class ApiConstants { public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine"; public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot"; public static final String TARGET_IQN = "targetiqn"; + public static final String TARIFF_ID = "tariffid"; + public static final String TARIFF_NAME = "tariffname"; public static final String TASKS_FILTER = "tasksfilter"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; @@ -608,6 +618,7 @@ public class ApiConstants { public static final String TENANT_NAME = "tenantname"; public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; + public static final String TO_CHECKPOINT_ID = "tocheckpointid"; public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; @@ -655,6 +666,7 @@ public class ApiConstants { public static final String VIRTUAL_MACHINE_STATE = "vmstate"; public static final String VIRTUAL_MACHINES = "virtualmachines"; public static final String USAGE_ID = "usageid"; + public static final String USAGE_NAME = "usagename"; public static final String USAGE_TYPE = "usagetype"; public static final String INCLUDE_TAGS = "includetags"; @@ -871,6 +883,7 @@ public class ApiConstants { public static final String IS_SOURCE_NAT = "issourcenat"; public static final String IS_STATIC_NAT = "isstaticnat"; public static final String ITERATIONS = "iterations"; + public static final String ITEMS = "items"; public static final String SORT_BY = "sortby"; public static final String CHANGE_CIDR = "changecidr"; public static final String PURPOSE = "purpose"; @@ -1356,6 +1369,13 @@ public class ApiConstants { public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String KEEP_MAC_ADDRESS_ON_PUBLIC_NIC = "keepmacaddressonpublicnic"; + + public static final String PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC = + "Indicates whether to use the same MAC address for the public NIC of VRs on the same network. If \"true\", when creating redundant routers or recreating" + + " a VR, CloudStack will use the same MAC address for the public NIC of all VRs. Otherwise, if \"false\", new public NICs will always have " + + " a new MAC address."; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java index 18c96c371591..1ee41ac86c22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java @@ -21,8 +21,11 @@ import javax.servlet.http.HttpSession; +import org.apache.cloudstack.context.CallContext; + import com.cloud.domain.Domain; import com.cloud.exception.CloudAuthenticationException; +import com.cloud.user.Account; import com.cloud.user.UserAccount; public interface ApiServerService { @@ -52,4 +55,20 @@ public ResponseObject loginUser(HttpSession session, String username, String pas String getDomainId(Map params); boolean isPostRequestsAndTimestampsEnforced(); + + AsyncCmdResult processAsyncCmd(BaseAsyncCmd cmdObj, Map params, CallContext ctx, Long callerUserId, Account caller) throws Exception; + + class AsyncCmdResult { + public final Long objectId; + public final String objectUuid; + public final BaseAsyncCmd asyncCmd; + public final long jobId; + + public AsyncCmdResult(Long objectId, String objectUuid, BaseAsyncCmd asyncCmd, long jobId) { + this.objectId = objectId; + this.objectUuid = objectUuid; + this.asyncCmd = asyncCmd; + this.jobId = jobId; + } + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java index 30baa71d6d85..b9b13dcfd884 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java @@ -20,7 +20,7 @@ public abstract class BaseAsyncCreateCustomIdCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.CUSTOM_ID, type = CommandType.STRING, description = "An optional field, in case you want to set a custom id to the resource. Allowed to Root Admins only") - private String customId; + protected String customId; public String getCustomId() { return customId; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java new file mode 100644 index 000000000000..c98bfb850529 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -0,0 +1,100 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.utils.EnumUtils; + +@APICommand(name = "createImageTransfer", + description = "Create image transfer for a disk in backup. This API is intended for testing only and is disabled by default.", + responseObject = ImageTransferResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + @Parameter(name = ApiConstants.VOLUME_ID, + type = CommandType.UUID, + entityType = VolumeResponse.class, + required = true, + description = "ID of the disk/volume") + private Long volumeId; + + @Parameter(name = ApiConstants.DIRECTION, + type = CommandType.STRING, + required = true, + description = "Direction of the transfer: upload, download") + private String direction; + + @Parameter(name = ApiConstants.FORMAT, + type = CommandType.STRING, + description = "Format for the image transfer: raw/cow. 'raw' will create an NBD backend. 'cow' will use the File backend." + + "For download, only the 'raw' format is supported. Default: raw") + private String format; + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + public ImageTransfer.Direction getDirection() { + return ImageTransfer.Direction.valueOf(direction); + } + + public ImageTransfer.Format getFormat() { + return EnumUtils.getEnum(ImageTransfer.Format.class, format); + } + + @Override + public void execute() { + ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java new file mode 100644 index 000000000000..d0e17e86d427 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -0,0 +1,85 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "deleteVirtualMachineCheckpoint", + description = "Delete a VM checkpoint. This API is intended for testing only and is disabled by default.", + responseObject = SuccessResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = "checkpointid", + type = CommandType.STRING, + required = true, + description = "Checkpoint ID") + private String checkpointId; + + public Long getVmId() { + return vmId; + } + + public String getCheckpointId() { + return checkpointId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setCheckpointId(String checkpointId) { + this.checkpointId = checkpointId; + } + + @Override + public void execute() { + boolean result = kvmBackupExportService.deleteVmCheckpoint(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java new file mode 100644 index 000000000000..45173f8668ee --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -0,0 +1,103 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; + +@APICommand(name = "finalizeBackup", + description = "Finalize a VM backup session. This API is intended for testing only and is disabled by default.", + responseObject = BackupResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class FinalizeBackupCmd extends BaseAsyncCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "ID of the backup") + private Long backupId; + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + Backup backup = kvmBackupExportService.finalizeBackup(this); + + if (backup == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Finalizing backup " + backupId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java new file mode 100644 index 000000000000..dfc43e233bf2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "finalizeImageTransfer", + description = "Finalize an image transfer. This API is intended for testing only and is disabled by default.", + responseObject = SuccessResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class FinalizeImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + required = true, + description = "ID of the image transfer") + private Long imageTransferId; + + public Long getImageTransferId() { + return imageTransferId; + } + + @Override + public void execute() { + boolean result = kvmBackupExportService.finalizeImageTransfer(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java new file mode 100644 index 000000000000..d810d21ab5f8 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -0,0 +1,81 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "listImageTransfers", + description = "List image transfers for a backup. This API is intended for testing only and is disabled by default.", + responseObject = ImageTransferResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class ListImageTransfersCmd extends BaseListCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + description = "ID of the Image Transfer") + private Long id; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + public Long getId() { + return id; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + List responses = kvmBackupExportService.listImageTransfers(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java new file mode 100644 index 000000000000..a61661e982de --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.KVMBackupExportService; + +@APICommand(name = "listVirtualMachineCheckpoints", + description = "List checkpoints for a VM. This API is intended for testing only and is disabled by default.", + responseObject = CheckpointResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class ListVmCheckpointsCmd extends BaseListCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() { + List responses = kvmBackupExportService.listVmCheckpoints(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return 0; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java new file mode 100644 index 000000000000..1bf6d45db049 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -0,0 +1,120 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; + +@APICommand(name = "startBackup", + description = "Start a VM backup session using pull mode backup-begin on the KVM host. This API is intended for testing only and is disabled by default.", + responseObject = BackupResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) + public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, + type = CommandType.STRING, + description = "the description for the backup") + private String description; + + public Long getVmId() { + return vmId; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + @Override + public void execute() { + try { + Backup backup = kvmBackupExportService.startBackup(this); + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public void create() { + Backup backup = kvmBackupExportService.createBackup(this); + + if (backup != null) { + setEntityId(backup.getId()); + setEntityUuid(backup.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Starting backup for Instance " + vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java index 6deaea22ac6c..d333a74fdb3b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java @@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd { description = "Name of the CA service provider, otherwise the default configured provider plugin will be used") private String provider; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, + description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " + + "Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false", + since = "4.23.0") + private Boolean forced; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -79,6 +85,10 @@ public String getProvider() { return provider; } + public boolean isForced() { + return forced != null && forced; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -90,7 +100,7 @@ public void execute() { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId()); } - boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider()); + boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced()); SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index f6f66415f533..055443934609 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -19,23 +19,23 @@ import java.util.ArrayList; import java.util.List; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.commons.lang3.StringUtils; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.Pair; @@ -94,6 +94,13 @@ public class ListCfgsByCmd extends BaseListCmd { description = "The ID of the Image Store to update the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + @Parameter(name = ApiConstants.GROUP, type = CommandType.STRING, description = "Lists configuration by group name (primarily used for UI)", since = "4.18.0") private String groupName; @@ -139,6 +146,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + public String getGroupName() { return groupName; } @@ -200,6 +211,9 @@ private void setScope(ConfigurationResponse cfgResponse) { if (getImageStoreId() != null){ cfgResponse.setScope("imagestore"); } + if (getManagementServerId() != null){ + cfgResponse.setScope("managementserver"); + } } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java index 2d511cff34db..3c3c36b29d7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java @@ -23,16 +23,16 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.framework.config.ConfigKey; - import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.user.Account; import com.cloud.utils.Pair; @@ -84,6 +84,13 @@ public class ResetCfgCmd extends BaseCmd { description = "The ID of the Image Store to reset the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -116,6 +123,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -149,6 +160,9 @@ public void execute() { if (getImageStoreId() != null) { response.setScope(ConfigKey.Scope.ImageStore.name()); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name()); + } response.setValue(cfg.second()); this.setResponseObject(response); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java index 97dee8f638af..9db9529dc8d8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java @@ -16,9 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.admin.config; -import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; @@ -29,13 +27,17 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.commons.lang3.StringUtils; import com.cloud.user.Account; +import com.cloud.utils.crypt.DBEncryptionUtil; @APICommand(name = "updateConfiguration", description = "Updates a configuration.", responseObject = ConfigurationResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -88,6 +90,13 @@ public class UpdateCfgCmd extends BaseCmd { validations = ApiArgValidator.PositiveNumber) private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,7 +121,7 @@ public Long getClusterId() { return clusterId; } - public Long getStoragepoolId() { + public Long getStoragePoolId() { return storagePoolId; } @@ -128,6 +137,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -182,7 +195,7 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getClusterId() != null) { response.setScope("cluster"); } - if (getStoragepoolId() != null) { + if (getStoragePoolId() != null) { response.setScope("storagepool"); } if (getAccountId() != null) { @@ -191,6 +204,9 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getDomainId() != null) { response.setScope("domain"); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name().toLowerCase()); + } return response; } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java index 6804e4355ca8..cb427e659495 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java @@ -87,6 +87,7 @@ private void setupResponse(final boolean result, final String resourceUuid) { final HostHAResponse response = new HostHAResponse(); response.setId(resourceUuid); response.setProvider(getHaProvider().toLowerCase()); + response.setStatus(result); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index abca619f82a7..4d6ef7419616 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public void execute() { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index e202dfad77ba..8f5e6c784d6e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -252,7 +252,7 @@ protected ListResponse getHostResponses() { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java index c085abd42c76..69bca7c79e21 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java @@ -49,9 +49,14 @@ public class UpdateHostCmd extends BaseCmd { @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, - description = "The ID of OS category to update the host with") + description = "the ID of OS category used to prioritize VMs with matching OS category during the allocation process. " + + "It cannot be used alongside the 'guestosrule' parameter.") private Long osCategoryId; + @Parameter(name = ApiConstants.GUEST_OS_RULE, type = CommandType.STRING, description = "the guest OS rule written in JavaScript to match with the OS of the VM." + + "It cannot be used alongside the 'oscategoryid' parameter.") + private String guestOsRule; + @Parameter(name = ApiConstants.ALLOCATION_STATE, type = CommandType.STRING, description = "Change resource state of host, valid values are [Enable, Disable]. Operation may failed if host in states not allowing Enable/Disable") @@ -96,6 +101,10 @@ public Long getOsCategoryId() { return osCategoryId; } + public String getGuestOsRule() { + return guestOsRule; + } + public String getAllocationState() { return allocationState; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java index 3427cef33661..a3dcf08c3a31 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java @@ -22,7 +22,7 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.UserResponse; - +import org.apache.cloudstack.api.ApiArgValidator; import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.UserAccount; @@ -35,7 +35,7 @@ public class GetUserCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user") + @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user", validations = {ApiArgValidator.NotNullOrEmpty}) private String apiKey; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java index e11d20d06466..50ff97ac676f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java @@ -85,6 +85,9 @@ public class AssignVMCmd extends BaseCmd { "In case no security groups are provided the Instance is part of the default security group.") private List securityGroupIdList; + // Internal flag to allow assignment without adding a network + private boolean skipNetwork = false; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -113,6 +116,34 @@ public List getSecurityGroupIdList() { return securityGroupIdList; } + public boolean isSkipNetwork() { + return skipNetwork; + } + + ///////////////////////////////////////////////////// + /////////////////// Setters ///////////////////////// + ///////////////////////////////////////////////////// + + public void setVirtualMachineId(Long virtualMachineId) { + this.virtualMachineId = virtualMachineId; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setSkipNetwork(boolean skipNetwork) { + this.skipNetwork = skipNetwork; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java index e64c8b3f46c6..b7c3b2ba41a6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java @@ -41,6 +41,19 @@ public class DeployVMCmdByAdmin extends DeployVMCmd implements AdminCmd { @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "Destination Cluster ID to deploy the Instance to - parameter available for root admin only", since = "4.13") private Long clusterId; + @Parameter(name = ApiConstants.BLANK_INSTANCE, + type = CommandType.BOOLEAN, + description = "Whether to create a blank instance without storage and network", + since = "4.23.0") + private Boolean blankInstance; + + // Internal flag to allow deploying instance with a given type + private String instanceType; + + ///////////////////////////////////////////////////// + ////////////////// Getters ////////////////////////// + ///////////////////////////////////////////////////// + public Long getPodId() { return podId; } @@ -48,4 +61,37 @@ public Long getPodId() { public Long getClusterId() { return clusterId; } + + @Override + public boolean isBlankInstance() { + return Boolean.TRUE.equals(blankInstance); + } + + @Override + public String getInstanceType() { + if (!isBlankInstance()) { + return null; + } + return instanceType; + } + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } + + public void setBlankInstance(boolean blankInstance) { + this.blankInstance = blankInstance; + } + + public void setInstanceType(String instanceType) { + this.instanceType = instanceType; + } + + public void setCustomId(String customId) { + this.customId = customId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java index 93d4b610b900..cbe6494d4004 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -27,4 +29,20 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance. Once destroyed, only the administrator can recover it.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd {} +public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd { + + @Parameter( name = ApiConstants.FORCED, + type = CommandType.BOOLEAN, + description = "Force destroy the Instance", + since = "4.23.0") + Boolean forced; + + @Override + public boolean isForced() { + return Boolean.TRUE.equals(forced); + } + + public void setForced(Boolean forced) { + this.forced = forced; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java index 28cd642e1a41..29b1e740b67e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -102,6 +103,16 @@ public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index ca60ea674fe3..8a4053aa15da 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -123,7 +123,12 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE @Override public ApiCommandResourceType getApiResourceType() { - return ApiCommandResourceType.Backup; + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return vmId; } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 67ad7c71503f..f6e17a2b3908 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; @@ -81,7 +82,7 @@ public class CreateBackupScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.QUIESCE_VM, type = CommandType.BOOLEAN, required = false, - description = "Quiesce the instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + + description = "Quiesce the Instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + "The filesystem is frozen before the backup starts and thawed immediately after. " + "Requires the instance to have the QEMU Guest Agent installed and running.", since = "4.21.0") @@ -139,4 +140,14 @@ public void execute() throws ServerApiException { public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java index dcf9f15b4dc5..62c2f03d8807 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -99,6 +100,16 @@ public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java index 4644687817df..c15e6f8de684 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -127,4 +128,14 @@ public String getEventType() { public String getEventDescription() { return "Restoring volume "+ volumeUuid + " from backup " + getResourceUuid(ApiConstants.BACKUP_ID) + " and attaching it to Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index cbf6df081b3b..ee5b8568e835 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -199,6 +199,11 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the network") private Long asNumber; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -286,6 +291,10 @@ public String getSourceNatIP() { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + @Override public boolean isDisplay() { if(displayNetwork == null) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java index 2e638f1e2f76..7b6841d60976 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java @@ -105,6 +105,11 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd implements UserCmd { @Parameter(name = ApiConstants.SOURCE_NAT_IP, type = CommandType.STRING, description = "IPV4 address to be assigned to the public interface of the network router. This address must already be acquired for this network", since = "4.19") private String sourceNatIP; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -186,6 +191,10 @@ public String getSourceNatIP() { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index 5c5c8776bce3..cd6baaf827bb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -193,6 +193,22 @@ public Boolean getGpuEnabled() { return gpuEnabled; } + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setCpuNumber(Integer cpuNumber) { + this.cpuNumber = cpuNumber; + } + + public void setMemory(Integer memory) { + this.memory = memory; + } + + public void setEncryptRoot(Boolean encryptRoot) { + this.encryptRoot = encryptRoot; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java index 6347c38811e8..f6ef955956f6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java @@ -100,6 +100,26 @@ public String getMacAddress() { return NetUtils.standardizeMacAddress(macaddr); } + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setNetworkId(Long netId) { + this.netId = netId; + } + + public void setIpaddr(String ipaddr) { + this.ipaddr = ipaddr; + } + + public void setMacAddress(String macaddr) { + this.macaddr = macaddr; + } + + public void setDhcpOptions(Map dhcpOptions) { + this.dhcpOptions = dhcpOptions; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8c29d7338b85..68b0821ba44d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -61,10 +61,10 @@ import com.cloud.network.Network.IpAddresses; import com.cloud.offering.DiskOffering; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.net.Dhcp; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; -import com.cloud.utils.net.Dhcp; public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { @@ -75,13 +75,13 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "availability zone for the virtual machine") - private Long zoneId; + protected Long zoneId; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName}) - private String name; + protected String name; @Parameter(name = ApiConstants.DISPLAY_NAME, type = CommandType.STRING, description = "an optional user generated name for the virtual machine") - private String displayName; + protected String displayName; @Parameter(name=ApiConstants.PASSWORD, type=CommandType.STRING, description="The password of the virtual machine. If null, a random password will be generated for the VM.", since="4.19.0.0") @@ -89,21 +89,21 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme //Owner information @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") - private String accountName; + protected String accountName; @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") - private Long domainId; + protected Long domainId; //Network information //@ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") - private List networkIds; + protected List networkIds; @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootType; + protected String bootType; @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootMode; + protected String bootMode; @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") private Boolean bootIntoSetup; @@ -138,7 +138,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which to deploy the virtual machine. " + "The parameter is required and respected only when hypervisor info is not set on the ISO/Template passed to the call") - private String hypervisor; + protected String hypervisor; @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + @@ -147,10 +147,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576) - private String userData; + protected String userData; @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") - private Long userdataId; + protected Long userdataId; @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18") private Map userdataDetails; @@ -160,7 +160,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String sshKeyPairName; @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs used to login to the virtual machine") - private List sshKeyPairNames; + protected List sshKeyPairNames; @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "destination Host ID to deploy the VM to - parameter available for root admin only") private Long hostId; @@ -168,7 +168,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.SECURITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups id that going to be applied to the virtual machine. " + "Should be passed only when vm is created from a zone with Basic Network support." + " Mutually exclusive with securitygroupnames parameter") - private List securityGroupIdList; + protected List securityGroupIdList; @ACL @Parameter(name = ApiConstants.SECURITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups names that going to be applied to the virtual machine." @@ -189,10 +189,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String macAddress; @Parameter(name = ApiConstants.KEYBOARD, type = CommandType.STRING, description = "an optional keyboard device type for the virtual machine. valid value can be one of de,de-ch,es,es-latam,fi,fr,fr-be,fr-ch,is,it,jp,nl-be,no,pt,uk,us") - private String keyboard; + protected String keyboard; @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Deploy vm for the project") - private Long projectId; + protected Long projectId; @Parameter(name = ApiConstants.START_VM, type = CommandType.BOOLEAN, description = "true if start vm after creating; defaulted to true if not specified") private Boolean startVm; @@ -200,7 +200,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups id that are going to be applied to the virtual machine." + " Mutually exclusive with affinitygroupnames parameter") - private List affinityGroupIdList; + protected List affinityGroupIdList; @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups names that are going to be applied to the virtual machine." @@ -208,10 +208,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private List affinityGroupNameList; @Parameter(name = ApiConstants.DISPLAY_VM, type = CommandType.BOOLEAN, since = "4.2", description = "an optional field, whether to the display the vm to the end user or not.", authorized = {RoleType.Admin}) - private Boolean displayVm; + protected Boolean displayVm; @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.3", description = "used to specify the custom parameters. 'extraconfig' is not allowed to be passed in details") - private Map details; + protected Map details; @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) private String deploymentPlanner; @@ -225,7 +225,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private Map dataDiskTemplateToDiskOfferingList; @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120) - private String extraConfig; + protected String extraConfig; @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") private Boolean copyImageTags; @@ -798,6 +798,11 @@ public IoDriverPolicy getIoDriverPolicy() { } return null; } + + public String getInstanceType() { + return null; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 050592b97a3b..0a943fab118d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -16,10 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Stream; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -40,6 +41,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts an Instance based on a service offering, disk offering, and Template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, @@ -88,6 +90,111 @@ public boolean isVolumeOrSnapshotProvided() { return volumeId != null || snapshotId != null; } + public boolean isBlankInstance() { + return false; + } + + + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setName(String name) { + this.name = name; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setNetworkIds(List networkIds) { + this.networkIds = networkIds; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public void setUserData(String userData) { + this.userData = userData; + } + + public void setKeyboard(String keyboard) { + this.keyboard = keyboard; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setDisplayVm(Boolean displayVm) { + this.displayVm = displayVm; + } + + public void setUserDataId(Long userDataId) { + this.userdataId = userDataId; + } + + public void setAffinityGroupIds(List ids) { + this.affinityGroupIdList = ids; + } + + public void setDetails(Map details) { + this.details = details; + } + + public void setExtraConfig(String extraConfig) { + this.extraConfig = extraConfig; + } + + public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { + this.dynamicScalingEnabled = dynamicScalingEnabled; + } + + public void setServiceOfferingId(Long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + public void setTemplateId(Long templateId) { + this.templateId = templateId; + } + + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public void setSshKeyPairNames(List sshKeyPairNames) { + this.sshKeyPairNames = sshKeyPairNames; + } + + public void setSecurityGroupList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + @Override public void execute() { UserVm result; @@ -132,7 +239,7 @@ public void execute() { @Override public void create() throws ResourceAllocationException { - if (Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { + if (!isBlankInstance() && Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { throw new CloudRuntimeException("Please provide only one of the following parameters - template ID, volume ID or snapshot ID"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index 9e2f2bcb72ce..aec0688f1779 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -90,6 +90,10 @@ public List getVolumeIds() { return volumeIds; } + public boolean isForced() { + return false; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index e3ad0502f454..a5f06e6ecbb3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -288,6 +288,14 @@ public boolean isCleanupExtraConfig() { return Boolean.TRUE.equals(cleanupExtraConfig); } + public void setId(Long id) { + this.id = id; + } + + public void setSecurityGroupIdList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java index f39853512281..f50abaf73c96 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java @@ -70,6 +70,21 @@ public Long getProjectid() { return projectid; } + ///////////////////////////////////////////////////// + /////////////////// Setter/////////////////////////// + ///////////////////////////////////////////////////// + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public void setProjectId(Long projectid) { + this.projectid = projectid; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 78de05648486..ec7a626fa156 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -114,7 +114,8 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "Storage pool ID to create the volume in. Cannot be used with the snapshotid parameter.", - authorized = {RoleType.Admin}) + authorized = {RoleType.Admin}, + since = "4.22.1") private Long storageId; ///////////////////////////////////////////////////// @@ -150,6 +151,10 @@ public Long getMaxIops() { } public Long getSnapshotId() { + if (storageId != null && snapshotId != null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Snapshot ID cannot be specified with the Storage ID."); + } return snapshotId; } @@ -163,7 +168,8 @@ private Long getProjectId() { public Long getStorageId() { if (snapshotId != null && storageId != null) { - throw new IllegalArgumentException("StorageId parameter cannot be specified with the SnapshotId parameter."); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Storage ID cannot be specified with the Snapshot ID."); } return storageId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java index 66a558abf982..f6e811da6055 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java @@ -77,6 +77,10 @@ public Long getVirtualMachineId() { return virtualMachineId; } + public void setId(Long id) { + this.id = id; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index 2adbbd664085..86309d7f28a5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -130,6 +130,11 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2") private Boolean useVrIpResolver; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private boolean keepMacAddressOnPublicNic = true; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -214,6 +219,10 @@ public boolean getUseVrIpResolver() { return BooleanUtils.toBoolean(useVrIpResolver); } + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java index f2327c9073f3..6cfdb895977d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java @@ -69,6 +69,11 @@ public class UpdateVPCCmd extends BaseAsyncCustomIdCmd implements UserCmd { since = "4.19") private String sourceNatIP; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -97,6 +102,10 @@ public String getSourceNatIP() { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index b855bfe40b8d..51fcaa9836e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -127,6 +127,18 @@ public class BackupResponse extends BaseResponse { @Param(description = "Indicates whether the VM from which the backup was taken is expunged or not", since = "4.22.0") private Boolean isVmExpunged; + @SerializedName(ApiConstants.FROM_CHECKPOINT_ID) + @Param(description = "Previous active checkpoint ID for incremental backups", since = "4.23.0") + private String fromCheckpointId; + + @SerializedName(ApiConstants.TO_CHECKPOINT_ID) + @Param(description = "Next checkpoint ID for incremental backups", since = "4.23.0") + private String toCheckpointId; + + @SerializedName(ApiConstants.HOST_ID) + @Param(description = "Host ID where the backup is running", since = "4.23.0") + private String hostId; + public String getId() { return id; } @@ -314,4 +326,28 @@ public void setVmOfferingRemoved(Boolean vmOfferingRemoved) { public void setVmExpunged(Boolean isVmExpunged) { this.isVmExpunged = isVmExpunged; } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + public String getFromCheckpointId() { + return this.fromCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + public String getToCheckpointId() { + return this.toCheckpointId; + } + + public void setHostId(String hostId) { + this.hostId = hostId; + } + + public String getHostId() { + return this.hostId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java new file mode 100644 index 000000000000..2bec7711064f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java @@ -0,0 +1,53 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class CheckpointResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the checkpoint ID") + private String id; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the checkpoint creation time") + private Date created; + + @SerializedName(ApiConstants.IS_ACTIVE) + @Param(description = "whether this is the active checkpoint") + private Boolean isActive; + + public void setId(String id) { + this.id = id; + } + + public void setCreated(Date created) { + this.created = created; + } + + public void setIsActive(Boolean isActive) { + this.isActive = isActive; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index 5d085d1bee05..f8e1af8d5b76 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -63,6 +63,10 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "The OS category name of the host") private String osCategoryName; + @SerializedName(ApiConstants.GUEST_OS_RULE) + @Param(description = "the guest OS rule") + private String guestOsRule; + @SerializedName(ApiConstants.IP_ADDRESS) @Param(description = "The IP address of the host") private String ipAddress; @@ -999,4 +1003,12 @@ public void setExtensionName(String extensionName) { public String getExtensionName() { return extensionName; } + + public String getGuestOsRule() { + return guestOsRule; + } + + public void setGuestOsRule(String guestOsRule) { + this.guestOsRule = guestOsRule; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java new file mode 100644 index 000000000000..15576e8f1012 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java @@ -0,0 +1,104 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.backup.ImageTransfer; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = ImageTransfer.class) +public class ImageTransferResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the image transfer") + private String id; + + @SerializedName("backupid") + @Param(description = "the backup ID") + private String backupId; + + @SerializedName("vmid") + @Param(description = "the VM ID") + private String vmId; + + @SerializedName(ApiConstants.VOLUME_ID) + @Param(description = "the disk/volume ID") + private String diskId; + + @SerializedName("devicename") + @Param(description = "the device name (vda, vdb, etc)") + private String deviceName; + + @SerializedName("transferurl") + @Param(description = "the transfer URL") + private String transferUrl; + + @SerializedName("phase") + @Param(description = "the transfer phase") + private String phase; + + @SerializedName("direction") + @Param(description = "the image transfer direction: upload / download") + private String direction; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the date created") + private Date created; + + public void setId(String id) { + this.id = id; + } + + public void setBackupId(String backupId) { + this.backupId = backupId; + } + + public void setVmId(String vmId) { + this.vmId = vmId; + } + + public void setDiskId(String diskId) { + this.diskId = diskId; + } + + public void setDeviceName(String deviceName) { + this.deviceName = deviceName; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java index 7c4b733a80f5..3a3663af2551 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java @@ -331,6 +331,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement @Param(description = "The BGP peers for the network", since = "4.20.0") private Set bgpPeers; + @SerializedName(ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, since = "4.23.0") + private Boolean keepMacAddressOnPublicNic; + public NetworkResponse() {} public Boolean getDisplayNetwork() { @@ -702,4 +706,8 @@ public void setIpv6Dns1(String ipv6Dns1) { public void setIpv6Dns2(String ipv6Dns2) { this.ipv6Dns2 = ipv6Dns2; } + + public void setKeepMacAddressOnPublicNic(Boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java index acfabb113502..34d50d5b9f92 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java @@ -185,6 +185,10 @@ public class VpcResponse extends BaseResponseWithAnnotations implements Controll @Param(description = "The BGP peers for the VPC", since = "4.20.0") private Set bgpPeers; + @SerializedName(ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, since = "4.23.0") + private Boolean keepMacAddressOnPublicNic; + public void setId(final String id) { this.id = id; } @@ -366,4 +370,8 @@ public void addBgpPeer(BgpPeerResponse bgpPeer) { } this.bgpPeers.add(bgpPeer); } + + public void setKeepMacAddressOnPublicNic(Boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 951af9180e7f..2d68f18b953f 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -30,8 +30,16 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { + String getFromCheckpointId(); + + String getToCheckpointId(); + + Long getCheckpointCreateTime(); + + Long getHostId(); + enum Status { - Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged + Allocated, Queued, BackingUp, ReadyForImageTransfer, FinalizingImageTransfer, BackedUp, Error, Failed, Restoring, Removed, Expunged } class Metric { diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java new file mode 100644 index 000000000000..e1153be3ae02 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ImageTransfer extends ControlledEntity, InternalIdentity { + long getDataCenterId(); + + public enum Direction { + upload, download + } + + public enum Format { + raw, + cow + } + + public enum Backend { + nbd, + file + } + + public enum Phase { + initializing, transferring, finished, failed + } + + String getUuid(); + + Long getBackupId(); + + long getVolumeId(); + + long getHostId(); + + String getTransferUrl(); + + Phase getPhase(); + + Direction getDirection(); + + Backend getBackend(); + + String getSignedTicketId(); +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java new file mode 100644 index 000000000000..e1affb311ec6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -0,0 +1,106 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.List; + +import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; +import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; +import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.utils.component.PluggableService; + +/** + * Service for Creating Backups and ImageTransfer sessions which will be consumed by an external orchestrator. + */ +public interface KVMBackupExportService extends Configurable, PluggableService { + + ConfigKey ImageTransferIdleTimeoutSeconds = new ConfigKey<>("Advanced", Integer.class, + "image.transfer.idle.timeout.seconds", + "600", + "Seconds since last completed HTTP request to an image transfer before the image server unregisters it (idle timeout).", + true, ConfigKey.Scope.Zone); + + ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Advanced", Boolean.class, + "expose.kvm.backup.export.service.apis", + "false", + "Enable to expose APIs for testing the KVM Backup Export Service.", + false, ConfigKey.Scope.Global); + /** + * Creates a backup session for a VM + */ + Backup createBackup(StartBackupCmd cmd); + + /** + * Start a backup session for a VM + * Creates a new checkpoint and starts NBD server for pull-mode backup + */ + Backup startBackup(StartBackupCmd cmd); + + /** + * Finalize a backup session + * Stops NBD server, updates checkpoint tracking, deletes old checkpoints + */ + Backup finalizeBackup(FinalizeBackupCmd cmd); + + /** + * Create an image transfer object for a disk + * Registers NBD endpoint with ImageIO (stubbed for POC) + */ + ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd); + + ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format); + + boolean cancelImageTransfer(long imageTransferId); + + /** + * Finalize an image transfer + * Marks transfer as complete (NBD is closed globally in finalize backup) + */ + boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd); + + boolean finalizeImageTransfer(long imageTransferId); + + /** + * List image transfers for a backup + */ + List listImageTransfers(ListImageTransfersCmd cmd); + + /** + * List checkpoints for a VM + */ + List listVmCheckpoints(ListVmCheckpointsCmd cmd); + + /** + * Delete a VM checkpoint (no-op for normal flow, kept for API parity) + */ + boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd); + + /** + * List Compatible Data Center Ids for the service + */ + List listCompatibleDataCenterIds(); +} diff --git a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java index b0fb1ac73c21..d2ebdc25f1bd 100644 --- a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java +++ b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; +import com.trilead.ssh2.Connection; + import org.apache.cloudstack.framework.ca.CAProvider; import org.apache.cloudstack.framework.ca.CAService; import org.apache.cloudstack.framework.ca.Certificate; @@ -39,7 +41,10 @@ public interface CAManager extends CAService, Configurable, PluggableService { ConfigKey CAProviderPlugin = new ConfigKey<>("Advanced", String.class, "ca.framework.provider.plugin", "root", - "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true); + "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " + + "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " + + "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " + + "Restart management server(s) when changed.", false); ConfigKey CertKeySize = new ConfigKey<>("Advanced", Integer.class, "ca.framework.cert.keysize", @@ -85,6 +90,12 @@ public interface CAManager extends CAService, Configurable, PluggableService { "The actual implementation will depend on the configured CA provider.", false); + ConfigKey CaInjectDefaultTruststore = new ConfigKey<>("Advanced", Boolean.class, + "ca.framework.inject.default.truststore", "true", + "When true, injects the CA provider's certificate into the JVM default truststore on management server startup. " + + "This allows outgoing HTTPS connections from the management server to trust servers with certificates signed by the configured CA. " + + "Restart management server(s) when changed.", false); + /** * Returns a list of available CA provider plugins * @return returns list of CAProvider @@ -130,12 +141,26 @@ public interface CAManager extends CAService, Configurable, PluggableService { boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider); /** - * Provisions certificate for given active and connected agent host + * Provisions certificate for given agent host. + * When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents). * @param host + * @param reconnect * @param provider + * @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs * @return returns success/failure as boolean */ - boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider); + boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced); + + /** + * Provisions certificate for a KVM host using an existing SSH connection. + * Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import. + * Used during host discovery and for forced re-provisioning when the NIO agent is unreachable. + * @param sshConnection active SSH connection to the KVM host + * @param agentIp IP address of the KVM host agent + * @param agentHostname hostname of the KVM host agent + * @param caProvider optional CA provider plugin name (null uses default) + */ + void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname, String caProvider); /** * Setups up a new keystore and generates CSR for a host diff --git a/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java b/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java index 21184de27a26..3e349cf0bd39 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java @@ -23,6 +23,10 @@ import org.apache.cloudstack.api.command.user.storage.sharedfs.ChangeSharedFSServiceOfferingCmd; import org.apache.cloudstack.api.command.user.storage.sharedfs.CreateSharedFSCmd; import org.apache.cloudstack.api.command.user.storage.sharedfs.DestroySharedFSCmd; +import org.apache.cloudstack.api.command.user.storage.sharedfs.ListSharedFSCmd; +import org.apache.cloudstack.api.command.user.storage.sharedfs.UpdateSharedFSCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SharedFSResponse; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; @@ -31,11 +35,6 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.VirtualMachineMigrationException; -import org.apache.cloudstack.api.command.user.storage.sharedfs.ListSharedFSCmd; -import org.apache.cloudstack.api.command.user.storage.sharedfs.UpdateSharedFSCmd; -import org.apache.cloudstack.api.response.SharedFSResponse; -import org.apache.cloudstack.api.response.ListResponse; - public interface SharedFSService { List getSharedFSProviders(); @@ -69,4 +68,10 @@ public interface SharedFSService { SharedFS recoverSharedFS(Long sharedFSId); void deleteSharedFS(Long sharedFSId); + + SharedFS getSharedFSByUuid(String uuid); + + SharedFS getSharedFSForVmId(long vmId); + + SharedFS updateSharedFSPostRestore(long sharedFsId, long volumeId); } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java new file mode 100644 index 000000000000..27bc4614e1b5 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class AssignVMCmdTest { + + @Test + public void test_setSkipNetwork_default() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + Object value = ReflectionTestUtils.getField(assignVMCmd, "skipNetwork"); + Assert.assertTrue(value instanceof Boolean); + Assert.assertFalse((Boolean) value); + } + + @Test + public void test_setSkipNetwork_set() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + assignVMCmd.setSkipNetwork(true); + Object value = ReflectionTestUtils.getField(assignVMCmd, "skipNetwork"); + Assert.assertTrue(value instanceof Boolean); + Assert.assertTrue((Boolean) value); + } + + @Test + public void test_isSkipNetwork_default() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + Assert.assertFalse(assignVMCmd.isSkipNetwork()); + } + + @Test + public void test_isSkipNetwork_set() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + ReflectionTestUtils.setField(assignVMCmd, "skipNetwork", true); + Assert.assertTrue(assignVMCmd.isSkipNetwork()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java new file mode 100644 index 000000000000..d82dc9f766f9 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class DeployVMCmdByAdminTest { + + @InjectMocks + private DeployVMCmdByAdmin cmd; + + @Test + public void testIsBlankInstance_default() { + assertFalse(cmd.isBlankInstance()); + } + + @Test + public void testIsBlankInstance_true() { + ReflectionTestUtils.setField(cmd, "blankInstance", true); + assertTrue(cmd.isBlankInstance()); + } + + @Test + public void testIsBlankInstance_false() { + ReflectionTestUtils.setField(cmd, "blankInstance", false); + assertFalse(cmd.isBlankInstance()); + } + + @Test + public void testSetBlankInstance_default() { + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNull(obj); + } + + @Test + public void testSetBlankInstance_true() { + cmd.setBlankInstance(true); + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNotNull(obj); + assertTrue((boolean)obj); + } + + @Test + public void testSetBlankInstance_false() { + cmd.setBlankInstance(false); + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNotNull(obj); + assertFalse((boolean)obj); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java index f7e3e38d9c3f..09d396e40237 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.vm; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -41,6 +42,7 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.NetworkService; import com.cloud.utils.db.EntityManager; import com.cloud.vm.VmDetailConstants; @@ -480,4 +482,146 @@ public void testGetDataDiskTemplateToDiskOfferingMapInvalidTemplateId() { }); assertTrue(thrownException.getMessage().contains("Unable to translate and find entity with datadisktemplateid")); } + + @Test + public void testSetServiceOfferingId() { + cmd.setServiceOfferingId(101L); + assertEquals(Long.valueOf(101L), cmd.getServiceOfferingId()); + } + + @Test + public void testSetTemplateId() { + cmd.setTemplateId(102L); + assertEquals(Long.valueOf(102L), cmd.getTemplateId()); + } + + @Test + public void testSetVolumeId() { + cmd.setVolumeId(103L); + assertEquals(Long.valueOf(103L), cmd.getVolumeId()); + } + + @Test + public void testSetSnapshotId() { + cmd.setSnapshotId(104L); + assertEquals(Long.valueOf(104L), cmd.getSnapshotId()); + } + + @Test + public void testSetZoneId() { + cmd.setZoneId(105L); + assertEquals(Long.valueOf(105L), cmd.getZoneId()); + } + + @Test + public void testSetName() { + cmd.setName("vm-name"); + assertEquals("vm-name", cmd.getName()); + } + + @Test + public void testSetDisplayName() { + cmd.setDisplayName("vm-display-name"); + assertEquals("vm-display-name", cmd.getDisplayName()); + } + + @Test + public void testSetAccountName() { + cmd.setAccountName("account-name"); + assertEquals("account-name", cmd.getAccountName()); + } + + @Test + public void testSetDomainId() { + cmd.setDomainId(106L); + assertEquals(Long.valueOf(106L), cmd.getDomainId()); + } + + @Test + public void testSetNetworkIds() { + List networkIds = Arrays.asList(11L, 12L); + cmd.setNetworkIds(networkIds); + assertEquals(networkIds, cmd.getNetworkIds()); + } + + @Test + public void testSetBootType() { + cmd.setBootType("UEFI"); + assertEquals(BootType.UEFI, cmd.getBootType()); + } + + @Test + public void testSetBootMode() { + cmd.setBootType("UEFI"); + cmd.setBootMode("SECURE"); + assertEquals(BootMode.SECURE, cmd.getBootMode()); + } + + @Test + public void testSetHypervisor() { + cmd.setHypervisor("KVM"); + assertEquals(HypervisorType.KVM, cmd.getHypervisor()); + } + + @Test + public void testSetUserData() { + cmd.setUserData("dXNlci1kYXRh"); + assertEquals("dXNlci1kYXRh", cmd.getUserData()); + } + + @Test + public void testSetKeyboard() { + cmd.setKeyboard("us"); + assertEquals("us", cmd.getKeyboard()); + } + + @Test + public void testSetProjectId() { + cmd.setProjectId(107L); + assertEquals(Long.valueOf(107L), ReflectionTestUtils.getField(cmd, "projectId")); + } + + @Test + public void testSetDisplayVm() { + cmd.setDisplayVm(Boolean.FALSE); + assertEquals(Boolean.FALSE, cmd.isDisplayVm()); + } + + @Test + public void testSetUserDataId() { + cmd.setUserDataId(108L); + assertEquals(Long.valueOf(108L), cmd.getUserdataId()); + } + + @Test + public void testSetAffinityGroupIds() { + List affinityGroupIds = Arrays.asList(21L, 22L); + cmd.setAffinityGroupIds(affinityGroupIds); + assertEquals(affinityGroupIds, cmd.getAffinityGroupIdList()); + } + + @Test + public void testSetDetails() { + Map details = new HashMap<>(); + details.put("key", "value"); + cmd.setDetails(details); + assertEquals(details, ReflectionTestUtils.getField(cmd, "details")); + } + + @Test + public void testSetExtraConfig() { + cmd.setExtraConfig("cpu-mode=host-passthrough"); + assertEquals("cpu-mode=host-passthrough", cmd.getExtraConfig()); + } + + @Test + public void testSetDynamicScalingEnabled() { + cmd.setDynamicScalingEnabled(Boolean.FALSE); + assertFalse(cmd.isDynamicScalingEnabled()); + } + + @Test + public void testIsBlankInstance() { + assertFalse(cmd.isBlankInstance()); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java index acb2dc685976..9d56cffd6718 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java @@ -93,7 +93,7 @@ public void testExecute() throws ResourceUnavailableException, InsufficientCapac responseGenerator = Mockito.mock(ResponseGenerator.class); cmd._responseGenerator = responseGenerator; Mockito.verify(_vpcService, Mockito.times(0)).updateVpc(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyBoolean()); } } diff --git a/client/pom.xml b/client/pom.xml index 7118f455ab5f..7d37225dbbb3 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -377,11 +377,6 @@ cloud-plugin-explicit-dedication ${project.version} - - org.apache.cloudstack - cloud-plugin-host-allocator-random - ${project.version} - org.apache.cloudstack cloud-plugin-outofbandmanagement-driver-ipmitool @@ -617,6 +612,11 @@ cloud-plugin-backup-nas ${project.version} + + org.apache.cloudstack + cloud-plugin-integrations-veeam-control-service + ${project.version} + org.apache.cloudstack cloud-plugin-integrations-kubernetes-service @@ -721,17 +721,17 @@ org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on ${cs.bcprov.version} org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${cs.bcprov.version} org.bouncycastle - bctls-jdk15on + bctls-jdk18on ${cs.bcprov.version} @@ -911,13 +911,13 @@ org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on false ${project.build.directory}/lib org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on false ${project.build.directory}/lib @@ -941,7 +941,7 @@ org.bouncycastle - bctls-jdk15on + bctls-jdk18on false ${project.build.directory}/lib @@ -976,9 +976,9 @@ org.apache.tomcat.embed:tomcat-embed-core org.apache.geronimo.specs:geronimo-servlet_3.0_spec org.apache.geronimo.specs:geronimo-javamail_1.4_spec - org.bouncycastle:bcprov-jdk15on - org.bouncycastle:bcpkix-jdk15on - org.bouncycastle:bctls-jdk15on + org.bouncycastle:bcprov-jdk18on + org.bouncycastle:bcpkix-jdk18on + org.bouncycastle:bctls-jdk18on com.mysql:mysql-connector-j org.apache.cloudstack:cloud-plugin-storage-volume-storpool org.apache.cloudstack:cloud-plugin-storage-volume-linstor diff --git a/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java index 5a26b22ec6a5..41e266784db8 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java @@ -38,6 +38,8 @@ public CheckOnHostAnswer(CheckOnHostCommand cmd, Boolean alive, String details) public CheckOnHostAnswer(CheckOnHostCommand cmd, String details) { super(cmd, false, details); + determined = false; + alive = false; } public boolean isDetermined() { @@ -47,5 +49,4 @@ public boolean isDetermined() { public boolean isAlive() { return alive; } - } diff --git a/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java b/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java index 94239f2900ef..72b5217604dc 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java @@ -24,7 +24,7 @@ public class CheckOnHostCommand extends Command { HostTO host; - boolean reportCheckFailureIfOneStorageIsDown; + boolean reportIfHeartBeatFailedForOneStoragePool; protected CheckOnHostCommand() { } @@ -34,17 +34,17 @@ public CheckOnHostCommand(Host host) { setWait(20); } - public CheckOnHostCommand(Host host, boolean reportCheckFailureIfOneStorageIsDown) { + public CheckOnHostCommand(Host host, boolean reportIfHeartBeatFailedForOneStoragePool) { this(host); - this.reportCheckFailureIfOneStorageIsDown = reportCheckFailureIfOneStorageIsDown; + this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool; } public HostTO getHost() { return host; } - public boolean isCheckFailedOnOneStorage() { - return reportCheckFailureIfOneStorageIsDown; + public boolean shouldReportIfHeartBeatFailedForOneStoragePool() { + return reportIfHeartBeatFailedForOneStoragePool; } @Override diff --git a/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java b/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java index dc63b1ee746d..e2953e9c7ad7 100644 --- a/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java @@ -30,6 +30,7 @@ public class ScaleVmCommand extends Command { Integer maxSpeed; long minRam; long maxRam; + private boolean limitCpuUseChange; public VirtualMachineTO getVm() { return vm; @@ -43,7 +44,7 @@ public int getCpus() { return cpus; } - public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse) { + public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse, Double cpuQuotaPercentage, boolean limitCpuUseChange) { super(); this.vmName = vmName; this.cpus = cpus; @@ -52,6 +53,8 @@ public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpee this.minRam = minRam; this.maxRam = maxRam; this.vm = new VirtualMachineTO(1L, vmName, null, cpus, minSpeed, maxSpeed, minRam, maxRam, null, null, false, limitCpuUse, null); + this.vm.setCpuQuotaPercentage(cpuQuotaPercentage); + this.limitCpuUseChange = limitCpuUseChange; } public void setCpus(int cpus) { @@ -102,6 +105,10 @@ public VirtualMachineTO getVirtualMachine() { return vm; } + public boolean getLimitCpuUseChange() { + return limitCpuUseChange; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java new file mode 100644 index 000000000000..34cf6d4ca34c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class CreateImageTransferAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public CreateImageTransferAnswer() { + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java new file mode 100644 index 000000000000..95b56c9a9c38 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -0,0 +1,94 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class CreateImageTransferCommand extends Command { + private String transferId; + private String exportName; + private String socket; + private String direction; + private String checkpointId; + private String file; + private ImageTransfer.Backend backend; + private int idleTimeoutSeconds; + + public CreateImageTransferCommand() { + } + + private CreateImageTransferCommand(String transferId, String direction, String socket, int idleTimeoutSeconds) { + this.transferId = transferId; + this.direction = direction; + this.socket = socket; + this.idleTimeoutSeconds = idleTimeoutSeconds; + } + + public CreateImageTransferCommand(String transferId, String direction, String exportName, String socket, String checkpointId, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); + this.backend = ImageTransfer.Backend.nbd; + this.exportName = exportName; + this.checkpointId = checkpointId; + } + + public CreateImageTransferCommand(String transferId, String direction, String socket, String file, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); + if (direction == ImageTransfer.Direction.download.toString()) { + throw new IllegalArgumentException("File backend is only supported for upload"); + } + this.backend = ImageTransfer.Backend.file; + this.file = file; + } + + public String getExportName() { + return exportName; + } + + public String getSocket() { + return socket; + } + + public String getFile() { + return file; + } + + public ImageTransfer.Backend getBackend() { + return backend; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getDirection() { + return direction; + } + + public String getCheckpointId() { + return checkpointId; + } + + public int getIdleTimeoutSeconds() { + return idleTimeoutSeconds; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java new file mode 100644 index 000000000000..81cf6c1abfcc --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java @@ -0,0 +1,60 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; + +public class DeleteVmCheckpointCommand extends Command { + private String vmName; + private String checkpointId; + private Map diskPathUuidMap; + private boolean stoppedVM; + + public DeleteVmCheckpointCommand() { + } + + public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map diskPathUuidMap, boolean stoppedVM) { + this.vmName = vmName; + this.checkpointId = checkpointId; + this.diskPathUuidMap = diskPathUuidMap; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getCheckpointId() { + return checkpointId; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java new file mode 100644 index 000000000000..84d9b1ff8186 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -0,0 +1,40 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class FinalizeImageTransferCommand extends Command { + private String transferId; + + public FinalizeImageTransferCommand() { + } + + public FinalizeImageTransferCommand(String transferId) { + this.transferId = transferId; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java new file mode 100644 index 000000000000..d7cbf097df90 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java @@ -0,0 +1,44 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartBackupAnswer extends Answer { + private Long checkpointCreateTime; + + public StartBackupAnswer() { + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details, Long checkpointCreateTime) { + super(cmd, success, details); + this.checkpointCreateTime = checkpointCreateTime; + } + + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java new file mode 100644 index 000000000000..6f1ed6834500 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -0,0 +1,91 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class StartBackupCommand extends Command { + private String vmName; + private String toCheckpointId; + private String fromCheckpointId; + private Long fromCheckpointCreateTime; + private String socket; + private Map diskPathUuidMap; + private boolean stoppedVM; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map diskPathPassphraseMap; + + public StartBackupCommand() { + } + + public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, Long fromCheckpointCreateTime, + String socket, Map diskPathUuidMap, Map passphrases, boolean stoppedVM) { + this.vmName = vmName; + this.toCheckpointId = toCheckpointId; + this.fromCheckpointId = fromCheckpointId; + this.fromCheckpointCreateTime = fromCheckpointCreateTime; + this.socket = socket; + this.diskPathUuidMap = diskPathUuidMap; + this.diskPathPassphraseMap = passphrases; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getToCheckpointId() { + return toCheckpointId; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public Long getFromCheckpointCreateTime() { + return fromCheckpointCreateTime; + } + + public String getSocket() { + return socket; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isIncremental() { + return fromCheckpointId != null && !fromCheckpointId.isEmpty(); + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + public Map getDiskPathPassphraseMap() { + return diskPathPassphraseMap; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java new file mode 100644 index 000000000000..d8c78d3c8807 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartNBDServerAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public StartNBDServerAnswer() { + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java new file mode 100644 index 000000000000..07e50f11fd37 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -0,0 +1,78 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class StartNBDServerCommand extends Command { + private String transferId; + private String exportName; + private String volumePath; + private String socket; + private String direction; + private String fromCheckpointId; + @LogLevel(LogLevel.Log4jLevel.Off) + private byte[] passphrase; + + public StartNBDServerCommand() { + } + + protected StartNBDServerCommand(String transferId, String exportName, String volumePath, String socket, String direction, String fromCheckpointId, byte[] passphrase) { + this.transferId = transferId; + this.socket = socket; + this.exportName = exportName; + this.volumePath = volumePath; + this.direction = direction; + this.fromCheckpointId = fromCheckpointId; + this.passphrase = passphrase; + } + + public String getExportName() { + return exportName; + } + + public String getSocket() { + return socket; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getVolumePath() { + return volumePath; + } + + public String getDirection() { + return direction; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public byte[] getPassphrase() { + return passphrase; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java new file mode 100644 index 000000000000..ce977f31e005 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java @@ -0,0 +1,30 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StopBackupAnswer extends Answer { + + public StopBackupAnswer() { + } + + public StopBackupAnswer(StopBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java new file mode 100644 index 000000000000..d3055021e9de --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java @@ -0,0 +1,52 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopBackupCommand extends Command { + private String vmName; + private Long vmId; + private Long backupId; + + public StopBackupCommand() { + } + + public StopBackupCommand(String vmName, Long vmId, Long backupId) { + this.vmName = vmName; + this.vmId = vmId; + this.backupId = backupId; + } + + public String getVmName() { + return vmName; + } + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java new file mode 100644 index 000000000000..d75168a22eb2 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java @@ -0,0 +1,46 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopNBDServerCommand extends Command { + private String transferId; + private String direction; + + public StopNBDServerCommand() { + } + + public StopNBDServerCommand(String transferId, String direction) { + this.transferId = transferId; + this.direction = direction; + } + + public String getTransferId() { + return transferId; + } + + public String getDirection() { + return direction; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/debian/changelog b/debian/changelog index 02251137e9d0..41f97748a0b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,13 +2,19 @@ cloudstack (4.23.0.0-SNAPSHOT) unstable; urgency=low * Update the version to 4.23.0.0-SNAPSHOT - -- the Apache CloudStack project Thu, 30 Oct 2025 19:23:55 +0530 + -- the Apache CloudStack project Fri, 22 May 2026 10:20:00 -0300 + +cloudstack (4.22.1.0) unstable; urgency=low + + * Update the version to 4.22.1.0 -cloudstack (4.23.0.0-SNAPSHOT-SNAPSHOT) unstable; urgency=low + -- the Apache CloudStack project Mon, 11 May 2026 20:26:07 +0530 - * Update the version to 4.23.0.0-SNAPSHOT-SNAPSHOT +cloudstack (4.22.0.0) unstable; urgency=low - -- the Apache CloudStack project Thu, Aug 28 11:58:36 2025 +0530 + * Update the version to 4.22.0.0 + + -- the Apache CloudStack project Thu, 30 Oct 2025 19:23:55 +0530 cloudstack (4.21.0.0) unstable; urgency=low diff --git a/debian/control b/debian/control index 2b8ce929c639..cdf663ef8906 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 57dc1b7bf728..5eca02f33d51 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -59,6 +59,8 @@ */ public interface VirtualMachineManager extends Manager { + String KVM_BLANK_VM_TEMPLATE_NAME = "kvm-blank-vm-template"; + ConfigKey ExecuteInSequence = new ConfigKey<>("Advanced", Boolean.class, "execute.in.sequence.hypervisor.commands", "false", "If set to true, start, stop, reboot, copy and migrate commands will be serialized on the agent side. If set to false the commands are executed in parallel. Default value is false.", false); @@ -181,15 +183,6 @@ void orchestrateStart(String vmUuid, Map pa void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); @@ -321,4 +314,8 @@ void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTemplate t ServiceOffering serviceOffering, Account systemAccount, DeploymentPlan plan) throws InsufficientServerCapacityException; + boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template); + + boolean isBlankInstance(VirtualMachineTemplate template); + } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index b7b548fb9407..030c1277efe2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -220,6 +220,11 @@ Network createGuestNetwork(long networkOfferingId, String name, String displayTe Boolean displayNetworkEnabled, String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + Network createGuestNetwork(long networkOfferingId, String name, String displayText, String gateway, String cidr, String vlanId, boolean bypassVlanOverlapCheck, String networkDomain, Account owner, + Long domainId, PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Boolean subdomainAccess, Long vpcId, String ip6Gateway, String ip6Cidr, + Boolean displayNetworkEnabled, String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, + String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize, boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + UserDataServiceProvider getPasswordResetProvider(Network network); UserDataServiceProvider getSSHKeyResetProvider(Network network); diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 4c81c7359f25..abaf6ea967d0 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -133,6 +133,20 @@ public interface CapacityManager { "capacity.calculate.workers", "1", "Number of worker threads to be used for capacities calculation", true); + ConfigKey KvmMemoryDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "kvm.memory.dynamic.scaling.capacity", "0", + "Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. " + + "The 'kvm.memory.dynamic.scaling.capacity' setting's value will be used to define the value of the " + + "'' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's memory capacity will be considered.", + true, ConfigKey.Scope.Cluster); + + ConfigKey KvmCpuDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "kvm.cpu.dynamic.scaling.capacity", "0", + "Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. " + + "The 'kvm.cpu.dynamic.scaling.capacity' setting's value will be used to define the value of the " + + "'' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's CPU cores capacity will be considered.", + true, ConfigKey.Scope.Cluster); + public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost); diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index 3ae94479cea5..53bfcce27038 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -75,10 +75,10 @@ public interface HighAvailabilityManager extends Manager { + " which are registered for the HA event that were successful and are now ready to be purged.", true, Cluster); - public static final ConfigKey KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false", + ConfigKey KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false", "Proceed fencing the host even the heartbeat failed for only one storage pool", false, ConfigKey.Scope.Zone); - public enum WorkType { + enum WorkType { Migration, // Migrating VMs off of a host. Stop, // Stops a VM for storage pool migration purposes. This should be obsolete now. CheckStop, // Checks if a VM has been stopped. diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 71ecc73f325f..db50a5134d8d 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -50,6 +50,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.hypervisor.KVMGuru; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -302,8 +303,8 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; @@ -575,7 +576,13 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t logger.debug("Allocating disks for {}", persistedVm); - allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + if (isBlankInstance(template)) { + logger.debug("Template is a dummy template for hypervisor {}, skipping volume allocation", hyperType); + return; + } else { + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + } + // Create new Volume context and inject event resource type, id and details to generate VOLUME.CREATE event for the ROOT disk. CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume); @@ -4017,19 +4024,6 @@ protected void runInContext() { } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -4469,11 +4463,6 @@ public void checkIfCanUpgrade(final VirtualMachine vmInstance, final ServiceOffe throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { @@ -5180,7 +5169,7 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old try { result = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); } catch (Exception ex) { - throw new RuntimeException("Unhandled exception", ex); + throw new RuntimeException("Unable to reconfigure VM.", ex); } if (result != null) { @@ -5193,22 +5182,29 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering oldServiceOffering, ServiceOffering newServiceOffering, boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException { - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + VMInstanceVO vm = _vmDao.findByUuid(vmUuid); HostVO hostVo = _hostDao.findById(vm.getHostId()); - Long clustedId = hostVo.getClusterId(); - Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clustedId); - Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clustedId); - boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clustedId); - boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clustedId); + Long clusterId = hostVo.getClusterId(); + Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clusterId); + Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clusterId); + boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clusterId); + boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clusterId); int minMemory = (int)(newServiceOffering.getRamSize() / (divideMemoryByOverprovisioning ? memoryOvercommitRatio : 1)); int minSpeed = (int)(newServiceOffering.getSpeed() / (divideCpuByOverprovisioning ? cpuOvercommitRatio : 1)); - ScaleVmCommand scaleVmCommand = - new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, - newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse()); + Double cpuQuotaPercentage = null; + if (newServiceOffering.getLimitCpuUse() && vm.getHypervisorType().equals(HypervisorType.KVM)) { + KVMGuru kvmGuru = (KVMGuru) _hvGuruMgr.getGuru(vm.getHypervisorType()); + cpuQuotaPercentage = kvmGuru.getCpuQuotaPercentage(minSpeed, hostVo.getSpeed()); + } + + boolean limitCpuUseChange = oldServiceOffering.getLimitCpuUse() != newServiceOffering.getLimitCpuUse(); + ScaleVmCommand scaleVmCommand = new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, newServiceOffering.getSpeed(), + minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, + newServiceOffering.getLimitCpuUse(), cpuQuotaPercentage, limitCpuUseChange); scaleVmCommand.getVirtualMachine().setId(vm.getId()); scaleVmCommand.getVirtualMachine().setUuid(vm.getUuid()); @@ -5237,16 +5233,20 @@ private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering old throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails())); } - upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering); + if (reconfiguringOnExistingHost) { + _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); + } + + boolean vmUpgraded = upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering); + if (vmUpgraded) { + vm = _vmDao.findById(vm.getId()); + } if (vm.getType().equals(VirtualMachine.Type.User)) { _userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE); } if (reconfiguringOnExistingHost) { - vm.setServiceOfferingId(oldServiceOffering.getId()); - _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); - vm.setServiceOfferingId(newServiceOffering.getId()); _capacityMgr.allocateVmCapacity(vm, false); } @@ -6714,4 +6714,18 @@ public void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTem vmProfile), DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); } } + + @Override + public boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template) { + return KVM_BLANK_VM_TEMPLATE_NAME.equals(template.getUniqueName()); + } + + @Override + public boolean isBlankInstance(VirtualMachineTemplate template) { + if (isBlankInstanceDefaultTemplate(template)) { + return true; + } + return Boolean.TRUE.equals( + MapUtils.getBoolean(CallContext.current().getContextParameters(), ApiConstants.BLANK_INSTANCE)); + } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 8639f006383f..964265cb873e 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.orchestration; import com.cloud.storage.Snapshot; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.template.VirtualMachineTemplate; import java.net.URL; @@ -292,9 +293,11 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow ServiceOfferingVO computeOffering = _serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + VMTemplateVO iso = _templateDao.findByIdIncludingRemoved(Long.valueOf(isoId)); + DiskOfferingInfo rootDiskOfferingInfo = new DiskOfferingInfo(); - if (diskOfferingId == null) { + if (diskOfferingId == null && !_itMgr.isBlankInstance(iso)) { throw new InvalidParameterValueException("Installing from ISO requires a disk offering to be specified for the root disk."); } DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); @@ -345,7 +348,7 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow HypervisorType hypervisorType = HypervisorType.valueOf(hypervisor); - _itMgr.allocate(vm.getInstanceName(), _templateDao.findByIdIncludingRemoved(new Long(isoId)), computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, + _itMgr.allocate(vm.getInstanceName(), iso, computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, networkIpMap, plan, hypervisorType, extraDhcpOptionMap, null, volume, snapshot); return vmEntity; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 7d455e7d6dc9..4262ee701aab 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -863,6 +863,7 @@ private static NetworkVO getNetworkVO(long id, final NetworkOffering offering, f vpcId, offering.isRedundantRouter(), predefined.getExternalId()); vo.setDisplayNetwork(isDisplayNetworkEnabled == null || isDisplayNetworkEnabled); vo.setStrechedL2Network(offering.isSupportingStrechedL2()); + vo.setKeepMacAddressOnPublicNic(predefined.getKeepMacAddressOnPublicNic()); return vo; } @@ -2724,7 +2725,7 @@ public Network createPrivateNetwork(final long networkOfferingId, final String n return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, null, owner, null, pNtwk, pNtwk.getDataCenterId(), ACLType.Account, null, vpcId, null, null, true, null, null, null, true, null, null, - null, null, null, null, null, null); + null, null, null, null, null, null, true); } @Override @@ -2735,10 +2736,25 @@ public Network createGuestNetwork(final long networkOfferingId, final String nam final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, + networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, + isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, + ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize, true); + } + + @Override + @DB + public Network createGuestNetwork(final long networkOfferingId, final String name, final String displayText, final String gateway, final String cidr, String vlanId, + boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, + final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, + final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, + String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, + Pair vrIfaceMTUs, Integer networkCidrSize, boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { // create Isolated/Shared/L2 network return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, - isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize); + isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, + ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize, keepMacAddressOnPublicNic); } @DB @@ -2747,7 +2763,8 @@ private Network createGuestNetwork(final long networkOfferingId, final String na final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork, String routerIp, String routerIpv6, final String ip4Dns1, final String ip4Dns2, - final String ip6Dns1, final String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + final String ip6Dns1, final String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize, + boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { final NetworkOfferingVO ntwkOff = _networkOfferingDao.findById(networkOfferingId); final DataCenterVO zone = _dcDao.findById(zoneId); @@ -3095,6 +3112,7 @@ public Network doInTransaction(final TransactionStatus status) { } } userNetwork.setNetworkCidrSize(networkCidrSize); + userNetwork.setKeepMacAddressOnPublicNic(keepMacAddressOnPublicNic); final List networks = setupNetwork(owner, ntwkOff, userNetwork, plan, name, displayText, true, domainId, aclType, subdomainAccessFinal, vpcId, isDisplayNetworkEnabled); Network network; diff --git a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml index 17c5002c718b..49c668f50e8b 100644 --- a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml +++ b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml @@ -88,6 +88,7 @@ + diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index 6cfd2608f5de..3d04f6679def 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -23,6 +23,7 @@ import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface ClusterDao extends GenericDao { @@ -61,4 +62,6 @@ public interface ClusterDao extends GenericDao { List listDistinctStorageAccessGroups(String name, String keyword); List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, HypervisorType hypervisorType, CPU.CPUArch arch); + + List listByZonesAndHypervisorType(List zoneIds, HypervisorType hypervisorType, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index c63af0a237ba..9b6f0f7ac3a6 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -20,6 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.cpu.CPU; @@ -38,6 +40,7 @@ import com.cloud.org.Grouping; import com.cloud.org.Managed; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; @@ -413,4 +416,19 @@ public List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, Hypervi } return customSearch(sc, null); } + + @Override + public List listByZonesAndHypervisorType(List zoneIds, HypervisorType hypervisorType, Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + sc.setParameters("hypervisorType", hypervisorType.toString()); + return listBy(sc, filter); + } } diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index d51b4eca0577..468caf1e2271 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -45,7 +45,7 @@ import com.cloud.cpu.CPU; import org.apache.cloudstack.util.CPUArchConverter; import org.apache.cloudstack.util.HypervisorTypeConverter; -import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; +import org.apache.cloudstack.utils.jsinterpreter.GenericRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.BooleanUtils; @@ -856,7 +856,8 @@ public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering) { } if (BooleanUtils.isTrue(this.getIsTagARule())) { - return TagAsRuleHelper.interpretTagAsRule(this.getHostTags().get(0), serviceOffering.getHostTag(), HostTagsDao.hostTagRuleExecutionTimeout.value()); + return GenericRuleHelper.interpretTagAsRule(this.getHostTags().get(0), serviceOffering.getHostTag(), HostTagsDao.hostTagRuleExecutionTimeout.value(), + HostTagsDao.hostTagRuleExecutionTimeout.key()); } if (StringUtils.isEmpty(serviceOffering.getHostTag())) { diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 090b019334f4..5c0d04fb2be8 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -32,12 +32,17 @@ import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.config.ConfigKey; /** * Data Access Object for server * */ public interface HostDao extends GenericDao, StateDao { + + ConfigKey guestOsRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "guest.os.rule.execution.timeout", "3000", "The maximum runtime, in milliseconds, " + + "to execute a guest OS rule; if it is reached, a timeout will happen.", true); + long countBy(long clusterId, ResourceState... states); Integer countAllByType(final Host.Type type); @@ -218,7 +223,9 @@ int countHostsByMsResourceStateTypeAndHypervisorType(long msId, List listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); + + List findHostsWithGuestOsRulesThatDidNotMatchOsOfGuestVm(String templateGuestOSName); List findClustersThatMatchHostTagRule(String computeOfferingTags); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 2d8fcca6cdb7..cd4423dfa269 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -37,7 +37,9 @@ import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.utils.jsinterpreter.GenericRuleHelper; import org.apache.commons.collections.CollectionUtils; import com.cloud.agent.api.VgpuTypesInfo; @@ -84,7 +86,7 @@ @DB @TableGenerator(name = "host_req_sq", table = "op_host", pkColumnName = "id", valueColumnName = "sequence", allocationSize = 1) -public class HostDaoImpl extends GenericDaoBase implements HostDao { //FIXME: , ExternalIdDao { +public class HostDaoImpl extends GenericDaoBase implements HostDao, Configurable { private static final String LIST_HOST_IDS_BY_HOST_TAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count " + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag,is_tag_a_rule) AS filtered " @@ -1520,11 +1522,13 @@ private List findHostIdsByHostTags(String hostTags){ } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + @Override + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { - if (TagAsRuleHelper.interpretTagAsRule(rule.getTag(), computeOfferingTags, HostTagsDao.hostTagRuleExecutionTimeout.value())) { + if (GenericRuleHelper.interpretTagAsRule(rule.getTag(), computeOfferingTags, HostTagsDao.hostTagRuleExecutionTimeout.value(), + HostTagsDao.hostTagRuleExecutionTimeout.key())) { result.add(findById(rule.getHostId())); } } @@ -1532,9 +1536,26 @@ public List findHostsWithTagRuleThatMatchComputeOferringTags(String comp return result; } + @Override + public List findHostsWithGuestOsRulesThatDidNotMatchOsOfGuestVm(String templateGuestOSName) { + List hostIdsWithGuestOsRule = _detailsDao.findByName(Host.GUEST_OS_RULE); + List hostsWithIncompatibleRules = new ArrayList<>(); + for (DetailVO guestOsRule : hostIdsWithGuestOsRule) { + if (!GenericRuleHelper.interpretGuestOsRule(guestOsRule.getValue(), templateGuestOSName, HostDao.guestOsRuleExecutionTimeout.value(), + HostDao.guestOsRuleExecutionTimeout.key())) { + logger.trace("The guest OS rule [{}] of the host with ID [{}] is incompatible with the OS of the VM.", + guestOsRule.getHostId(), guestOsRule.getValue()); + hostsWithIncompatibleRules.add(findById(guestOsRule.getHostId())); + } + } + logger.trace("The hosts with the following IDs [{}] are incompatible with the VM considering their guest OS rule.", + hostsWithIncompatibleRules); + return hostsWithIncompatibleRules; + } + public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } @@ -1984,4 +2005,14 @@ public List listDistinctStorageAccessGroups(String name, String keyword) return customSearch(sc, null); } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {guestOsRuleExecutionTimeout}; + } + + @Override + public String getConfigComponentName() { + return HostDaoImpl.class.getSimpleName(); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index fdca6e43f00f..8406f80ed09d 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -24,6 +24,7 @@ import com.cloud.network.Network.GuestType; import com.cloud.network.Network.State; import com.cloud.network.Networks.TrafficType; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.fsm.StateDao; @@ -96,8 +97,13 @@ public interface NetworkDao extends GenericDao, StateDao serviceProviderMap); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType, Filter filter); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType); + List listByZonesTrafficTypeAndOwners(List zoneIds, final TrafficType trafficType, + List accountIds, List domainIds, Filter filter); + void setCheckForGc(long networkId); int getNetworkCountByNetworkOffId(long networkOfferingId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 9f7ffabac930..f3ece4879040 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -19,6 +19,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -29,9 +30,9 @@ import javax.inject.Inject; import javax.persistence.TableGenerator; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.network.Network; @@ -63,6 +64,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.SequenceFetcher; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @Component @@ -632,12 +634,46 @@ public List listBy(final long accountId, final long dataCenterId, fin } @Override - public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType, Filter filter) { final SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("datacenter", zoneId); sc.setParameters("trafficType", trafficType); - return listBy(sc, null); + return listBy(sc, filter); + } + + @Override + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + return listByZoneAndTrafficType(zoneId, trafficType, null); + } + + @Override + public List listByZonesTrafficTypeAndOwners(List zoneIds, final TrafficType trafficType, + List accountIds, List domainIds, Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + sb.and("trafficType", sb.entity().getTrafficType(), Op.EQ); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + sc.setParameters("trafficType", trafficType); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + return listBy(sc, filter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java index 02abaacd854e..f2572ba91c21 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java @@ -203,9 +203,13 @@ public class NetworkVO implements Network { @Column(name = "private_mtu") Integer privateMtu; + @Column(name = "keep_mac_address_on_public_nic") + private boolean keepMacAddressOnPublicNic = true; + @Transient Integer networkCidrSize; + public NetworkVO() { uuid = UUID.randomUUID().toString(); } @@ -773,4 +777,13 @@ public Integer getNetworkCidrSize() { public void setNetworkCidrSize(Integer networkCidrSize) { this.networkCidrSize = networkCidrSize; } + + @Override + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + + public void setKeepMacAddressOnPublicNic(boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java index ccba6bb18893..606bdaaaa7a7 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java @@ -19,9 +19,21 @@ import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.utils.db.GenericDao; +import java.util.Date; +import java.util.List; + public interface PublicIpQuarantineDao extends GenericDao { PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId); PublicIpQuarantineVO findByIpAddress(String publicIpAddress); + + /** + * Returns a list of public IP addresses that are actively quarantined at the specified date and the previous owner differs from the specified user. + * + * @param userId used to check against the IP's previous owner; + * @param date used to check if the quarantine is active; + * @return a list of PublicIpQuarantineVOs. + */ + List listQuarantinedIpAddressesToUser(Long userId, Date date); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java index a1b789b8a46b..0c47a0d36e30 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java @@ -26,6 +26,8 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import java.util.Date; +import java.util.List; @Component public class PublicIpQuarantineDaoImpl extends GenericDaoBase implements PublicIpQuarantineDao { @@ -33,6 +35,8 @@ public class PublicIpQuarantineDaoImpl extends GenericDaoBase ipAddressSearchBuilder; + private SearchBuilder quarantinedIpAddressesSearch; + @Inject IPAddressDao ipAddressDao; @@ -47,8 +51,16 @@ public void init() { publicIpAddressByIdSearch.join("quarantineJoin", ipAddressSearchBuilder, ipAddressSearchBuilder.entity().getId(), publicIpAddressByIdSearch.entity().getPublicIpAddressId(), JoinBuilder.JoinType.INNER); + quarantinedIpAddressesSearch = createSearchBuilder(); + quarantinedIpAddressesSearch.and("previousOwnerId", quarantinedIpAddressesSearch.entity().getPreviousOwnerId(), SearchCriteria.Op.NEQ); + quarantinedIpAddressesSearch.and(); + quarantinedIpAddressesSearch.op("removedIsNull", quarantinedIpAddressesSearch.entity().getRemoved(), SearchCriteria.Op.NULL); + quarantinedIpAddressesSearch.and("endDate", quarantinedIpAddressesSearch.entity().getEndDate(), SearchCriteria.Op.GT); + quarantinedIpAddressesSearch.cp(); + ipAddressSearchBuilder.done(); publicIpAddressByIdSearch.done(); + quarantinedIpAddressesSearch.done(); } @Override @@ -68,4 +80,14 @@ public PublicIpQuarantineVO findByIpAddress(String publicIpAddress) { return findOneBy(sc, filter); } + + @Override + public List listQuarantinedIpAddressesToUser(Long userId, Date date) { + SearchCriteria sc = quarantinedIpAddressesSearch.create(); + + sc.setParameters("previousOwnerId", userId); + sc.setParameters("endDate", date); + + return searchIncludingRemoved(sc, null, false, false); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java index 793334731095..2b18bcbd87bd 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java @@ -31,4 +31,6 @@ public interface SecurityGroupDao extends GenericDao { List findByAccountAndNames(Long accountId, String... names); int removeByAccountId(long accountId); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java index 019cf5fec462..ffb45629b625 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java @@ -129,4 +129,14 @@ public boolean expunge(Long id) { txn.commit(); return result; } + + @Override + public List listByIds(List ids) { + SearchBuilder idsSearch = createSearchBuilder(); + idsSearch.and("ids", idsSearch.entity().getId(), SearchCriteria.Op.IN); + idsSearch.done(); + SearchCriteria sc = idsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java index e942eadb8ffb..742d3f2f82ee 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java @@ -108,6 +108,9 @@ public class VpcVO implements Vpc { @Column(name = "use_router_ip_resolver") boolean useRouterIpResolver = false; + @Column(name = "keep_mac_address_on_public_nic") + private boolean keepMacAddressOnPublicNic = true; + @Transient boolean rollingRestart = false; @@ -321,4 +324,13 @@ public boolean useRouterIpAsResolver() { public void setUseRouterIpResolver(boolean useRouterIpResolver) { this.useRouterIpResolver = useRouterIpResolver; } + + @Override + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + + public void setKeepMacAddressOnPublicNic(boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java index 9beea0037444..ea97f663768f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java @@ -30,7 +30,7 @@ public interface DiskOfferingDao extends GenericDao { List listAllBySizeAndProvisioningType(long size, Storage.ProvisioningType provisioningType); - List findCustomDiskOfferings(); + List listCustomDiskOfferings(); List listByStorageTag(String tag); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java index 4ca3fe9f12ac..44e6551da17d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage.dao; +import static org.apache.cloudstack.query.QueryService.SortKeyAscending; + import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -33,6 +35,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.utils.db.Attribute; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -138,13 +141,14 @@ public List listAllBySizeAndProvisioningType(long size, Storage. } @Override - public List findCustomDiskOfferings() { + public List listCustomDiskOfferings() { SearchBuilder sb = createSearchBuilder(); sb.and("customized", sb.entity().isCustomized(), SearchCriteria.Op.EQ); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("customized", true); - return listBy(sc); + Filter searchFilter = new Filter(DiskOfferingVO.class, "sortKey", SortKeyAscending.value()); + return listBy(sc, searchFilter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 4c9f906b68a9..aec06d6d0003 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -106,4 +106,6 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findActiveSystemTemplateByHypervisorArchAndUrlPath(HypervisorType hypervisorType, CPU.CPUArch arch, String urlPathSuffix); + + VMTemplateVO findByAccountAndName(Long accountId, String templateName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 9b5d0edc599d..8c6e3fe0983f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -945,4 +945,12 @@ public boolean updateState( } return rows > 0; } + + @Override + public VMTemplateVO findByAccountAndName(Long accountId, String templateName) { + SearchCriteria sc = NameAccountIdSearch.create(); + sc.setParameters("name", templateName); + sc.setParameters("accountId", accountId); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 717e3e782f2f..7d566a1ea01e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -46,6 +46,8 @@ public interface VolumeDao extends GenericDao, StateDao findByInstanceAndType(long id, Volume.Type vType); + List findByInstanceAndNotStates(long id, Volume.State...states); + List findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index fce4d1f7233d..91f1c7f5eb69 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -208,6 +208,17 @@ public List findByInstanceAndType(long id, Type vType) { return listBy(sc); } + @Override + public List findByInstanceAndNotStates(long id, Volume.State...states) { + SearchBuilder sb = createSearchBuilder(); + sb.and("instanceId", sb.entity().getInstanceId(), Op.EQ); + sb.and("state", sb.entity().getState(), Op.NIN); + SearchCriteria sc = sb.create(); + sc.setParameters("instanceId", id); + sc.setParameters("state", (Object[]) states); + return listBy(sc); + } + @Override public List findIncludingRemovedByInstanceAndType(long id, Type vType) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index bacb09b98793..5efaea40a943 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -20,11 +20,13 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.response.ResourceTagResponse; + import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import org.apache.cloudstack.api.response.ResourceTagResponse; public interface ResourceTagDao extends GenericDao { @@ -60,4 +62,13 @@ public interface ResourceTagDao extends GenericDao { void removeByResourceIdAndKey(long resourceId, ResourceObjectType resourceType, String key); List listByResourceUuid(String resourceUuid); + + List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter); + + ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, String value); + + List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, String key); + } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index cc9d99e6ab16..22c7b7b2ee5b 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -16,19 +16,22 @@ // under the License. package com.cloud.tags.dao; -import java.util.List; -import java.util.Set; -import java.util.Map; import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; @@ -120,4 +123,62 @@ public List listByResourceUuid(String resourceUuid) { sc.setParameters("resourceUuid", resourceUuid); return listBy(sc); } + + @Override + public List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter) { + GenericSearchBuilder sb = createSearchBuilder(String.class); + sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getValue()); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key + "%"); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + return customSearch(sc, filter); + } + + @Override + public ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, + String value) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.and("value", sb.entity().getValue(), Op.EQ); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key + "%"); + sc.setParameters("value", value); + return findOneBy(sc); + } + + @Override + public List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, + String key) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("resourceId", sb.entity().getResourceId(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("resourceId", resourceId); + sc.setParameters("key", key + "%"); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 292bafefbb65..2acb4138d234 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -1073,7 +1073,7 @@ protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplate } Hypervisor.HypervisorType hypervisorType = templateDetails.getHypervisorType(); updateSystemVMEntries(templateId, hypervisorType); - updateConfigurationParams(hypervisorType, templateDetails.getName(), zoneId); + updateConfigurationParams(hypervisorType, templateVO.getName(), zoneId); } protected void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 96a1b6e3bd16..7767b7512b91 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -26,6 +26,8 @@ public interface NicDao extends GenericDao { List listByVmId(long instanceId); + int countByVmId(long instanceId); + List listByVmIdOrderByDeviceId(long instanceId); List listIpAddressInNetwork(long networkConfigId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index 78966a09e97c..6202dabf730e 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -126,6 +126,13 @@ public List listByVmId(long instanceId) { return listBy(sc); } + @Override + public int countByVmId(long instanceId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("instance", instanceId); + return getCount(sc); + } + @Override public List listByVmIdOrderByDeviceId(long instanceId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 4fd3e729e0d2..1a5b8cedd9ea 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -197,4 +197,6 @@ List searchRemovedByRemoveDate(final Date startDate, final Date en List listDeleteProtectedVmsByAccountId(long accountId); List listDeleteProtectedVmsByDomainIds(Set domainIds); + + List listIdsByHostIdForVolumeStats(long hostIds); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 6ffa7cd59626..ae1e838649ba 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -358,7 +358,8 @@ protected void init() { IdsPowerStateSelectSearch.entity().getPowerHostId(), IdsPowerStateSelectSearch.entity().getPowerState(), IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(), - IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime()); + IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime(), + IdsPowerStateSelectSearch.entity().getState()); IdsPowerStateSelectSearch.done(); CountByOfferingId = createSearchBuilder(Integer.class); @@ -1105,10 +1106,14 @@ public Map updatePowerState( private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) { State instanceState = instance.getState(); + if (instanceState == null) { + logger.warn("VM {} has null instance state during power state sync check, treating as out of sync", instance); + return false; + } if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running) || (powerState == VirtualMachine.PowerState.PowerOn && instanceState == State.Stopped)) { HostVO instanceHost = hostDao.findById(instance.getHostId()); - HostVO powerHost = powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); + HostVO powerHost = instance.getHostId() != null && powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); logger.debug("VM: {} on host: {} and power host : {} is in {} state, but power state is {}", instance, instanceHost, powerHost, instanceState, powerState); return false; @@ -1331,4 +1336,20 @@ public List listDeleteProtectedVmsByDomainIds(Set domainIds) Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); return listBy(sc, filter); } + + @Override + public List listIdsByHostIdForVolumeStats(long hostId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.selectFields(sb.entity().getId()); + sb.and().op("host", sb.entity().getHostId(), SearchCriteria.Op.EQ); + sb.or().op("hostNull", sb.entity().getHostId(), Op.NULL); + sb.and("lastHost", sb.entity().getLastHostId(), SearchCriteria.Op.EQ); + sb.cp(); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("host", hostId); + sc.setParameters("lastHost", hostId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 0f8a10fb7be6..d589f9e6bef8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -103,6 +103,18 @@ public class BackupVO implements Backup { @Column(name = "backup_schedule_id") private Long backupScheduleId; + @Column(name = "from_checkpoint_id") + private String fromCheckpointId; + + @Column(name = "to_checkpoint_id") + private String toCheckpointId; + + @Column(name = "checkpoint_create_time") + private Long checkpointCreateTime; + + @Column(name = "host_id") + private Long hostId; + @Transient Map details; @@ -288,4 +300,40 @@ public Long getBackupScheduleId() { public void setBackupScheduleId(Long backupScheduleId) { this.backupScheduleId = backupScheduleId; } + + @Override + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + @Override + public String getToCheckpointId() { + return toCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + @Override + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } + + @Override + public Long getHostId() { + return hostId; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java new file mode 100644 index 000000000000..ec9b927b63e4 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -0,0 +1,242 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name = "image_transfer") +public class ImageTransferVO implements ImageTransfer { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "backup_id") + private Long backupId; + + @Column(name = "volume_id") + private long volumeId; + + @Column(name = "host_id") + private long hostId; + + @Column(name = "socket") + private String socket; + + @Column(name = "file") + private String file; + + @Column(name = "transfer_url") + private String transferUrl; + + @Enumerated(value = EnumType.STRING) + @Column(name = "phase") + private Phase phase; + + @Enumerated(value = EnumType.STRING) + @Column(name = "direction") + private Direction direction; + + @Enumerated(value = EnumType.STRING) + @Column(name = "backend") + private Backend backend; + + @Column(name = "signed_ticket_id") + private String signedTicketId; + + @Column(name = "account_id") + Long accountId; + + @Column(name = "domain_id") + Long domainId; + + @Column(name = "data_center_id") + Long dataCenterId; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = "updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date updated; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public ImageTransferVO() { + } + + private ImageTransferVO(String uuid, long volumeId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this.uuid = uuid; + this.volumeId = volumeId; + this.hostId = hostId; + this.phase = phase; + this.direction = direction; + this.accountId = accountId; + this.domainId = domainId; + this.dataCenterId = dataCenterId; + this.created = new Date(); + } + + public ImageTransferVO(String uuid, Long backupId, long volumeId, long hostId, String socket, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.backupId = backupId; + this.socket = socket; + this.backend = Backend.nbd; + } + + public ImageTransferVO(String uuid, long volumeId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.file = file; + this.backend = Backend.file; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public Long getBackupId() { + return backupId; + } + + public void setBackupId(long backupId) { + this.backupId = backupId; + } + + @Override + public long getVolumeId() { + return volumeId; + } + + public void setVolumeId(long volumeId) { + this.volumeId = volumeId; + } + + @Override + public long getHostId() { + return hostId; + } + + public void setHostId(long hostId) { + this.hostId = hostId; + } + + public void setSocket(String socket) { + this.socket = socket; + } + + @Override + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + @Override + public Phase getPhase() { + return phase; + } + + public void setPhase(Phase phase) { + this.phase = phase; + this.updated = new Date(); + } + + @Override + public Direction getDirection() { + return direction; + } + + public void setDirection(Direction direction) { + this.direction = direction; + } + + @Override + public Backend getBackend() { + return backend; + } + + @Override + public String getSignedTicketId() { + return signedTicketId; + } + + public void setSignedTicketId(String signedTicketId) { + this.signedTicketId = signedTicketId; + } + + @Override + public Class getEntityType() { + return ImageTransfer.class; + } + + @Override + public String getName() { + return null; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public long getDataCenterId() { + return dataCenterId; + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java new file mode 100644 index 000000000000..9a9fbeb8814e --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -0,0 +1,36 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.ImageTransferVO; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDao; + +public interface ImageTransferDao extends GenericDao { + List listByBackupId(Long backupId); + ImageTransferVO findByUuid(String uuid); + ImageTransferVO findByVolume(Long volumeId); + ImageTransferVO findUnfinishedByVolume(Long volumeId); + List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); + List listByZonesAndOwners(List zoneIds, List accountIds, List domainIds, + Filter filter); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java new file mode 100644 index 000000000000..2e36f924e4ef --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -0,0 +1,136 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.Collections; +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class ImageTransferDaoImpl extends GenericDaoBase implements ImageTransferDao { + + private SearchBuilder backupIdSearch; + private SearchBuilder uuidSearch; + private SearchBuilder volumeSearch; + private SearchBuilder volumeUnfinishedSearch; + private SearchBuilder phaseDirectionSearch; + + public ImageTransferDaoImpl() { + } + + @PostConstruct + protected void init() { + backupIdSearch = createSearchBuilder(); + backupIdSearch.and("backupId", backupIdSearch.entity().getBackupId(), SearchCriteria.Op.EQ); + backupIdSearch.done(); + + uuidSearch = createSearchBuilder(); + uuidSearch.and("uuid", uuidSearch.entity().getUuid(), SearchCriteria.Op.EQ); + uuidSearch.done(); + + volumeSearch = createSearchBuilder(); + volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeSearch.done(); + + volumeUnfinishedSearch = createSearchBuilder(); + volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeUnfinishedSearch.and("phase", volumeUnfinishedSearch.entity().getPhase(), SearchCriteria.Op.NEQ); + volumeUnfinishedSearch.done(); + + phaseDirectionSearch = createSearchBuilder(); + phaseDirectionSearch.and("phase", phaseDirectionSearch.entity().getPhase(), SearchCriteria.Op.EQ); + phaseDirectionSearch.and("direction", phaseDirectionSearch.entity().getDirection(), SearchCriteria.Op.EQ); + phaseDirectionSearch.done(); + } + + @Override + public List listByBackupId(Long backupId) { + SearchCriteria sc = backupIdSearch.create(); + sc.setParameters("backupId", backupId); + return listBy(sc); + } + + @Override + public ImageTransferVO findByUuid(String uuid) { + SearchCriteria sc = uuidSearch.create(); + sc.setParameters("uuid", uuid); + return findOneBy(sc); + } + + @Override + public ImageTransferVO findByVolume(Long volumeId) { + SearchCriteria sc = volumeSearch.create(); + sc.setParameters("volumeId", volumeId); + return findOneBy(sc); + } + + @Override + public ImageTransferVO findUnfinishedByVolume(Long volumeId) { + SearchCriteria sc = volumeUnfinishedSearch.create(); + sc.setParameters("volumeId", volumeId); + sc.setParameters("phase", ImageTransferVO.Phase.finished.toString()); + return findOneBy(sc); + } + + @Override + public List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction) { + SearchCriteria sc = phaseDirectionSearch.create(); + sc.setParameters("phase", phase); + sc.setParameters("direction", direction); + return listBy(sc); + } + + @Override + public List listByZonesAndOwners(List zoneIds, List accountIds, List domainIds, + Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + + return listBy(sc, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java index 4735202a7623..82ba9445b25c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java @@ -29,4 +29,6 @@ public interface SharedFSDao extends GenericDao, StateDao listSharedFSToBeDestroyed(Date date); SharedFSVO findSharedFSByNameAccountDomain(String name, Long accountId, Long domainId); + + SharedFSVO findByVm(long vmId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java index da6220716715..dd23787f982f 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java @@ -114,4 +114,14 @@ public SharedFSVO findSharedFSByNameAccountDomain(String name, Long accountId, L sc.setParameters("domainId", domainId); return findOneBy(sc); } + + @Override + public SharedFSVO findByVm(long vmId) { + SearchBuilder sb = createSearchBuilder(); + sb.and("vmId", sb.entity().getVmId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("vmId", vmId); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index ad3722577c27..26181d3fce0d 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -116,6 +116,7 @@ + @@ -271,6 +272,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 858c46a7c1ee..fbb2fd079f9c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -19,7 +19,6 @@ -- Schema upgrade from 4.21.0.0 to 4.22.0.0 --; - -- health check status as enum CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 54baf226ac43..2f104568c140 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,3 +18,9 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; + +-- Entries remaining on `cloud`.`resource_reservation` during the upgrade process are stale, so delete them. +-- This script was added to normalize volume/primary storage reservations that got stuck due to a bug on VM deployment, +-- but it is more interesting to introduce a smarter logic to clean these stale reservations in the future without the need +-- for upgrades (for instance, by having a heartbeat_time column for the reservations and automatically cleaning old entries). +DELETE FROM `cloud`.`resource_reservation`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index c0feb06e76ac..2d25b3355d8e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -127,3 +127,84 @@ CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_tariff_usage` ( PRIMARY KEY (`id`), CONSTRAINT `fk_quota_tariff_usage__tariff_id` FOREIGN KEY (`tariff_id`) REFERENCES `cloud_usage`.`quota_tariff` (`id`), CONSTRAINT `fk_quota_tariff_usage__quota_usage_id` FOREIGN KEY (`quota_usage_id`) REFERENCES `cloud_usage`.`quota_usage` (`id`)); + +-- Add the 'keep_mac_address_on_public_nic' column to the 'cloud.networks' and 'cloud.vpc' tables +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.networks', 'keep_mac_address_on_public_nic', 'TINYINT(1) NOT NULL DEFAULT 1'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc', 'keep_mac_address_on_public_nic', 'TINYINT(1) NOT NULL DEFAULT 1'); + +-- Creates the 'kvm.memory.dynamic.scaling.capacity' and, for already active ACS environments, +-- initializes it with the value of the setting 'vm.serviceoffering.ram.size.max' +INSERT INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `updated`, `scope`, `is_dynamic`, `group_id`, `subgroup_id`, `display_text`, `description`) +SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.memory.dynamic.scaling.capacity', `cfg`.`value`, 0, NULL, 4, 1, 6, 27, + 'KVM memory dynamic scaling capacity', 'Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. The ''kvm.memory.dynamic.scaling.capacity'' setting''s value will be used to define the value of the '''' element of domain XMLs. If it is set to a value less than or equal to ''0'', then the host''s memory capacity will be considered.' +FROM `cloud`.`configuration` `cfg` +WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.memory.dynamic.scaling.capacity') + AND `cfg`.`name` = 'vm.serviceoffering.ram.size.max'; + +-- Creates the 'kvm.cpu.dynamic.scaling.capacity' and, for already active ACS environments, +-- initializes it with the value of the setting 'vm.serviceoffering.cpu.cores.max' +INSERT INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `updated`, `scope`, `is_dynamic`, `group_id`, `subgroup_id`, `display_text`, `description`) +SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.cpu.dynamic.scaling.capacity', `cfg`.`value`, 0, NULL, 4, 1, 6, 27, + 'KVM CPU dynamic scaling capacity', 'Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. The ''kvm.cpu.dynamic.scaling.capacity'' setting''s value will be used to define the value of the '''' element of domain XMLs. If it is set to a value less than or equal to ''0'', then the host''s CPU cores capacity will be considered.' +FROM `cloud`.`configuration` `cfg` +WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.cpu.dynamic.scaling.capacity') + AND `cfg`.`name` = 'vm.serviceoffering.cpu.cores.max'; + +-- Remove stale realhostip.com default values; domain has been dead since ~2015. +UPDATE `cloud`.`configuration` + SET value = NULL + WHERE name IN ('consoleproxy.url.domain', 'secstorage.ssl.cert.domain') + AND value IN ('realhostip.com', '*.realhostip.com'); + +-- Add management_server_details table to allow ManagementServer scope configs +CREATE TABLE IF NOT EXISTS `management_server_details` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `management_server_id` bigint unsigned NOT NULL COMMENT 'management server the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user', + PRIMARY KEY (`id`), + CONSTRAINT `fk_management_server_details__management_server_id` FOREIGN KEY `fk_management_server_details__management_server_id`(`management_server_id`) REFERENCES `mshost`(`id`) ON DELETE CASCADE, + KEY `i_management_server_details__name__value` (`name`(128),`value`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Add checkpoint tracking fields to backups table for incremental backup support +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Previous active checkpoint id for incremental backups"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for the next incremental backup"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); + +-- Create image_transfer table for per-disk image transfers +CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) NOT NULL COMMENT 'uuid', + `account_id` bigint unsigned NOT NULL COMMENT 'Account ID', + `domain_id` bigint unsigned NOT NULL COMMENT 'Domain ID', + `data_center_id` bigint unsigned NOT NULL COMMENT 'Data Center ID', + `backup_id` bigint unsigned COMMENT 'Backup ID', + `volume_id` bigint unsigned NOT NULL COMMENT 'Volume ID', + `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', + `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', + `file` varchar(255) COMMENT 'File for the file backend', + `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', + `socket` varchar(255) COMMENT 'Unix socket for nbd backend', + `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', + `backend` varchar(20) NOT NULL COMMENT 'Backend: nbd, file', + `progress` int COMMENT 'Transfer progress percentage (0-100)', + `signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO', + `created` datetime NOT NULL COMMENT 'date created', + `updated` datetime COMMENT 'date updated if not null', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + CONSTRAINT `fk_image_transfer__backup_id` FOREIGN KEY (`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__volume_id` FOREIGN KEY (`volume_id`) REFERENCES `volumes`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE, + INDEX `i_image_transfer__backup_id`(`backup_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +--- Quota resource statement +INSERT INTO cloud.role_permissions (uuid, role_id, rule, permission, sort_order) +SELECT uuid(), role_id, 'quotaResourceStatement', permission, sort_order +FROM cloud.role_permissions rp +WHERE rule = 'quotaStatement' AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaResourceStatement'); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql index d9f4e2671595..255ce866c427 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql @@ -64,6 +64,7 @@ SELECT guest_os_category.id guest_os_category_id, guest_os_category.uuid guest_os_category_uuid, guest_os_category.name guest_os_category_name, + (SELECT `value` FROM `cloud`.`host_details` `hd` WHERE `hd`.`host_id` = `cloud`.`host`.`id` AND `hd`.`name` = 'guest.os.rule') AS `guest_os_rule`, mem_caps.used_capacity memory_used_capacity, mem_caps.reserved_capacity memory_reserved_capacity, cpu_caps.used_capacity cpu_used_capacity, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index c169cbce2170..d77ba27689be 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -56,6 +56,7 @@ SELECT `vm_instance`.`display_vm` AS `display_vm`, `vm_instance`.`delete_protection` AS `delete_protection`, `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_display_name`, `vm_instance`.`pod_id` AS `pod_id`, `host_pod_ref`.`uuid` AS `pod_uuid`, `vm_instance`.`private_ip_address` AS `private_ip_address`, diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java index 9445efeb089c..6f153727ab76 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java @@ -41,6 +41,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; @@ -113,6 +114,69 @@ public void testListPoolIdsByVolumeCount_without_cluster_details() throws SQLExc verify(preparedStatementMock, times(1)).executeQuery(); } + @Test + public void findByInstanceAndNotState_queriesWithInstanceIdAndExcludedStates() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + + volumeDao.findByInstanceAndNotStates(42L, Volume.State.Ready); + + Mockito.verify(sc).setParameters("instanceId", 42L); + Mockito.verify(sc).setParameters("state", (Object[]) new Volume.State[]{Volume.State.Ready}); + } + + @Test + public void findByInstanceAndNotStates_withMultipleExcludedStates_passesAllStatesToCriteria() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + + volumeDao.findByInstanceAndNotStates(7L, Volume.State.Destroy, Volume.State.Expunged); + + Mockito.verify(sc).setParameters("instanceId", 7L); + Mockito.verify(sc).setParameters("state", + (Object[]) new Volume.State[]{Volume.State.Destroy, Volume.State.Expunged}); + } + + @Test + public void findByInstanceAndNotStates_returnsResultFromDao() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + VolumeVO vol = Mockito.mock(VolumeVO.class); + Mockito.doReturn(List.of(vol)).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + + List result = volumeDao.findByInstanceAndNotStates(1L, Volume.State.Ready); + + Assert.assertEquals(1, result.size()); + Assert.assertSame(vol, result.get(0)); + } + + @Test + public void findByInstanceAndNotStates_noMatchingVolumes_returnsEmptyList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + + List result = volumeDao.findByInstanceAndNotStates(99L, Volume.State.Ready); + + Assert.assertTrue(result.isEmpty()); + } + @Test public void testSearchRemovedByVmsNoVms() { Assert.assertTrue(CollectionUtils.isEmpty(volumeDao.searchRemovedByVms( @@ -141,5 +205,4 @@ public void testSearchRemovedByVms() { Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), Mockito.eq(false)); } - } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index 8028e78c9073..51db952eb613 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -570,18 +570,19 @@ public void updateRegisteredTemplateDetails_UpdatesTemplateSuccessfully() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + String templateName = "templateName"; + when(templateVO.getName()).thenReturn(templateName); GuestOSVO guestOS = Mockito.mock(GuestOSVO.class); when(templateDetails.getGuestOs()).thenReturn("Debian"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(templateDetails.getName()).thenReturn("templateName"); when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); when(guestOSDao.findOneByDisplayName("Debian")).thenReturn(guestOS); when(guestOS.getId()).thenReturn(10L); when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); @@ -590,7 +591,7 @@ public void updateRegisteredTemplateDetails_UpdatesTemplateSuccessfully() { verify(vmTemplateDao).update(templateVO.getId(), templateVO); verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); } @Test @@ -620,16 +621,17 @@ public void updateRegisteredTemplateDetails_SkipsGuestOSUpdateWhenNotFound() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + String templateName = "templateName"; + when(templateVO.getName()).thenReturn(templateName); when(templateDetails.getGuestOs()).thenReturn("NonExistentOS"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(templateDetails.getName()).thenReturn("templateName"); when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); when(guestOSDao.findOneByDisplayName("NonExistentOS")).thenReturn(null); when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); @@ -637,7 +639,7 @@ public void updateRegisteredTemplateDetails_SkipsGuestOSUpdateWhenNotFound() { verify(vmTemplateDao).update(templateVO.getId(), templateVO); verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); } @Test diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 436f991afbd1..8731e8791ddd 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -395,9 +395,9 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { } // Find out if the volume is at state of download_in_progress on secondary storage - VolumeDataStoreVO volumeStore = _volumeStoreDao.findByVolume(volume.getId()); - if (volumeStore != null) { - if (volumeStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { + VolumeDataStoreVO volumeOnImageStore = _volumeStoreDao.findByVolume(volume.getId()); + if (volumeOnImageStore != null) { + if (volumeOnImageStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { String msg = String.format("Volume: %s is currently being uploaded; can't delete it.", volume); logger.debug(msg); result.setSuccess(false); @@ -416,10 +416,10 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { if (!volumeExistsOnPrimary(vol)) { // not created on primary store - if (volumeStore == null) { + if (volumeOnImageStore == null) { // also not created on secondary store if (logger.isDebugEnabled()) { - logger.debug("Marking volume that was never created as destroyed: " + vol); + logger.debug("Marking volume that was never created as destroyed: {}", vol); } VMTemplateVO template = templateDao.findById(vol.getTemplateId()); if (template != null && !template.isDeployAsIs()) { @@ -435,11 +435,21 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { if (volume.getDataStore().getRole() == DataStoreRole.Image) { // no need to change state in volumes table volume.processEventOnly(Event.DestroyRequested); + if (volumeOnImageStore == null) { + logger.debug("Volume {} doesn't exist on image store, no need to delete", vol); + future.complete(result); + return future; + } } else if (volume.getDataStore().getRole() == DataStoreRole.Primary) { if (vol.getState() == Volume.State.Expunging) { logger.info("Volume {} is already in Expunging, retrying", volume); } volume.processEvent(Event.ExpungeRequested); + if (!volumeExistsOnPrimary(vol)) { + logger.debug("Volume {} doesn't exist on primary storage, no need to delete", vol); + future.complete(result); + return future; + } } DeleteVolumeContext context = new DeleteVolumeContext<>(null, vo, future); @@ -460,13 +470,11 @@ public void ensureVolumeIsExpungeReady(long volumeId) { private boolean volumeExistsOnPrimary(VolumeVO vol) { Long poolId = vol.getPoolId(); - if (poolId == null) { return false; } PrimaryDataStore primaryStore = dataStoreMgr.getPrimaryDataStore(poolId); - if (primaryStore == null) { return false; } @@ -476,8 +484,7 @@ private boolean volumeExistsOnPrimary(VolumeVO vol) { } String volumePath = vol.getPath(); - - if (volumePath == null || volumePath.trim().isEmpty()) { + if (StringUtils.isBlank(volumePath)) { return false; } diff --git a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java index 7c5692564c99..c9c48dbb179f 100644 --- a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java +++ b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java @@ -119,10 +119,10 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) { byte[] decodedUserData = null; // If GET, use 4K. If POST, support up to 1M. - if (httpmethod.equals(BaseCmd.HTTPMethod.GET)) { - decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); - } else if (httpmethod.equals(BaseCmd.HTTPMethod.POST)) { + if (BaseCmd.HTTPMethod.POST.equals(httpmethod)) { decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_POST_LENGTH, BaseCmd.HTTPMethod.POST); + } else { + decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); } // Re-encode so that the '=' paddings are added if necessary since 'isBase64' does not require it, but python does on the VR. diff --git a/framework/cluster/pom.xml b/framework/cluster/pom.xml index 2dd28e8e628f..75bcaf9a7ddf 100644 --- a/framework/cluster/pom.xml +++ b/framework/cluster/pom.xml @@ -48,6 +48,12 @@ cloud-api ${project.version} + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + compile + diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java new file mode 100644 index 000000000000..fcaa2a22e341 --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "management_server_details") +public class ManagementServerHostDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "management_server_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + @Column(name = "display") + private boolean display = true; + + public ManagementServerHostDetailVO(long poolId, String name, String value, boolean display) { + this.resourceId = poolId; + this.name = name; + this.value = value; + this.display = display; + } + + public ManagementServerHostDetailVO() { + } + + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return display; + } +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java new file mode 100644 index 000000000000..24fd60d21b3c --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.cluster.ManagementServerHostDetailVO; +import com.cloud.utils.db.GenericDao; + +public interface ManagementServerHostDetailsDao extends GenericDao, ResourceDetailsDao { +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java new file mode 100644 index 000000000000..5865bee0926b --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + +import com.cloud.cluster.ManagementServerHostDetailVO; + +public class ManagementServerHostDetailsDaoImpl extends ResourceDetailsDaoBase implements ManagementServerHostDetailsDao, ScopedConfigStorage { + + public ManagementServerHostDetailsDaoImpl() { + } + + @Override + public ConfigKey.Scope getScope() { + return ConfigKey.Scope.ManagementServer; + } + + @Override + public String getConfigValue(long id, String key) { + ManagementServerHostDetailVO vo = findDetail(id, key); + return vo == null ? null : vo.getValue(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new ManagementServerHostDetailVO(resourceId, key, value, display)); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index 1422338ddc99..f6fd08b6da2c 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -1663,14 +1663,14 @@ public Extension getExtensionForCluster(long clusterId) { public List getExtensionReservedResourceDetails(long extensionId) { ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS); - if (detailsVO == null || !StringUtils.isNotBlank(detailsVO.getValue())) { - return Collections.emptyList(); - } List reservedDetails = new ArrayList<>(); - String[] parts = detailsVO.getValue().split(","); - for (String part : parts) { - if (StringUtils.isNotBlank(part)) { - reservedDetails.add(part.trim()); + if (detailsVO != null && StringUtils.isNotBlank(detailsVO.getValue())) { + String[] parts = detailsVO.getValue().split(","); + for (String part : parts) { + String trimmedPart = part.trim(); + if (StringUtils.isNotBlank(trimmedPart)) { + reservedDetails.add(trimmedPart); + } } } addInbuiltExtensionReservedResourceDetails(extensionId, reservedDetails); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 926280bfeade..c334e91feb84 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -68,4 +68,6 @@ public interface AsyncJobDao extends GenericDao { // Returns the number of pending jobs for the given Management server msids. // NOTE: This is the msid and NOT the id long countPendingNonPseudoJobs(Long... msIds); + + List listPendingJobIdsForAccount(long accountId); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 81cc5d4f2a8c..d8385e9aecd1 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -299,4 +299,14 @@ public long countPendingJobs(String havingInfo, String... cmds) { List results = customSearch(sc, null); return results.get(0); } + + @Override + public List listPendingJobIdsForAccount(long accountId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.selectFields(sb.entity().getId()); + SearchCriteria sc = sb.create(); + sc.setParameters("accountId", accountId); + return customSearch(sc, null); + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index 816144aa2f16..5afef8bc95b6 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -32,6 +33,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.quota.activationrule.presetvariables.Configuration; import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; @@ -43,9 +47,11 @@ import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaTariffDao; +import org.apache.cloudstack.quota.dao.QuotaTariffUsageDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.apache.cloudstack.usage.UsageUnitTypes; @@ -86,7 +92,8 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { private QuotaBalanceDao _quotaBalanceDao; @Inject private ConfigurationDao _configDao; - + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; @Inject protected PresetVariableHelper presetVariableHelper; @@ -150,8 +157,9 @@ protected void processQuotaBalanceForAccount(AccountVO accountVo, List> periods = accountQuotaUsages.stream() @@ -215,7 +223,7 @@ protected BigDecimal retrieveBalanceForUsageCalculation(long accountId, long dom logger.debug(String.format("Persisting the first quota balance [%s] for account [%s].", firstBalance, accountToString)); _quotaBalanceDao.saveQuotaBalance(firstBalance); } else { - QuotaBalanceVO lastRealBalance = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, startDate); + QuotaBalanceVO lastRealBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, startDate); if (lastRealBalance == null) { logger.warn("Account [{}] has quota usage entries, however it does not have a quota balance.", accountToString); @@ -244,7 +252,7 @@ protected void saveQuotaAccount(long accountId, BigDecimal aggregatedUsage, Date } protected BigDecimal aggregateCreditBetweenDates(Long accountId, Long domainId, Date startDate, Date endDate, String accountToString) { - List creditsReceived = _quotaBalanceDao.findCreditBalance(accountId, domainId, startDate, endDate); + List creditsReceived = _quotaBalanceDao.findCreditBalances(accountId, domainId, startDate, endDate); logger.debug("Account [{}] has [{}] credit entries before [{}].", accountToString, creditsReceived.size(), DateUtil.displayDateInTimezone(usageAggregationTimeZone, endDate)); @@ -310,14 +318,14 @@ protected List createQuotaUsagesAccordingToQuotaTariffs(AccountVO String accountToString = account.reflectionToString(); logger.info("Calculating quota usage of [{}] usage records for account [{}].", usageRecords.size(), accountToString); - List> pairsUsageAndQuotaUsage = new ArrayList<>(); + Map>> mapUsageAndQuotaUsage = new LinkedHashMap<>(); - try (JsInterpreter jsInterpreter = new JsInterpreter(QuotaConfig.QuotaActivationRuleTimeout.value())) { + try (JsInterpreter jsInterpreter = new JsInterpreter(QuotaConfig.QuotaActivationRuleTimeout.value(), QuotaConfig.QuotaActivationRuleTimeout.key())) { for (UsageVO usageRecord : usageRecords) { int usageType = usageRecord.getUsageType(); if (!shouldCalculateUsageRecord(account, usageRecord)) { - pairsUsageAndQuotaUsage.add(new Pair<>(usageRecord, null)); + mapUsageAndQuotaUsage.put(usageRecord, null); continue; } @@ -325,18 +333,31 @@ protected List createQuotaUsagesAccordingToQuotaTariffs(AccountVO List quotaTariffs = pairQuotaTariffsPerUsageTypeAndHasActivationRule.first(); boolean hasAnyQuotaTariffWithActivationRule = pairQuotaTariffsPerUsageTypeAndHasActivationRule.second(); - BigDecimal aggregatedQuotaTariffsValue = aggregateQuotaTariffsValues(usageRecord, quotaTariffs, hasAnyQuotaTariffWithActivationRule, jsInterpreter, accountToString); + Map aggregatedQuotaTariffsAndValues = aggregateQuotaTariffsValues(usageRecord, + quotaTariffs, hasAnyQuotaTariffWithActivationRule, jsInterpreter, accountToString); + BigDecimal aggregatedQuotaTariffsValue = aggregatedQuotaTariffsAndValues.values().stream().reduce(BigDecimal.ZERO, BigDecimal::add); + logger.debug("The aggregation of the quota tariffs of account [{}] resulted in [{}] for the usage record [{}].", + account, aggregatedQuotaTariffsValue, usageRecord); QuotaUsageVO quotaUsage = createQuotaUsageAccordingToUsageUnit(usageRecord, aggregatedQuotaTariffsValue, accountToString); + if (quotaUsage == null) { + mapUsageAndQuotaUsage.put(usageRecord, null); + continue; + } - pairsUsageAndQuotaUsage.add(new Pair<>(usageRecord, quotaUsage)); + List quotaTariffUsages = new ArrayList<>(); + for (Map.Entry entry : aggregatedQuotaTariffsAndValues.entrySet()) { + QuotaTariffUsageVO quotaTariffUsage = createQuotaTariffUsage(usageRecord, entry.getKey(), entry.getValue()); + quotaTariffUsages.add(quotaTariffUsage); + } + mapUsageAndQuotaUsage.put(usageRecord, new Pair<>(quotaUsage, quotaTariffUsages)); } } catch (Exception e) { logger.error(String.format("Failed to calculate the quota usage for account [%s] due to [%s].", accountToString, e.getMessage()), e); return new ArrayList<>(); } - return persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsages(pairsUsageAndQuotaUsage); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsages(mapUsageAndQuotaUsage)); } protected boolean shouldCalculateUsageRecord(AccountVO accountVO, UsageVO usageRecord) { @@ -348,31 +369,41 @@ protected boolean shouldCalculateUsageRecord(AccountVO accountVO, UsageVO usageR return true; } - protected List persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsages(List> pairsUsageAndQuotaUsage) { + protected List persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsages(Map>> mapUsageAndQuotaTariffUsage) { List quotaUsages = new ArrayList<>(); - for (Pair pairUsageAndQuotaUsage : pairsUsageAndQuotaUsage) { - UsageVO usageVo = pairUsageAndQuotaUsage.first(); + for (Map.Entry>> usageAndTariffUsage : mapUsageAndQuotaTariffUsage.entrySet()) { + UsageVO usageVo = usageAndTariffUsage.getKey(); usageVo.setQuotaCalculated(1); _usageDao.persistUsage(usageVo); - QuotaUsageVO quotaUsageVo = pairUsageAndQuotaUsage.second(); - if (quotaUsageVo != null) { - _quotaUsageDao.persistQuotaUsage(quotaUsageVo); - quotaUsages.add(quotaUsageVo); + Pair> pairUsageAndTariffUsages = usageAndTariffUsage.getValue(); + if (pairUsageAndTariffUsages != null) { + QuotaUsageVO quotaUsage = pairUsageAndTariffUsages.first(); + _quotaUsageDao.persistQuotaUsage(quotaUsage); + quotaUsages.add(quotaUsage); + + persistQuotaTariffUsages(pairUsageAndTariffUsages.second(), quotaUsage.getId()); } } return quotaUsages; } - protected BigDecimal aggregateQuotaTariffsValues(UsageVO usageRecord, List quotaTariffs, boolean hasAnyQuotaTariffWithActivationRule, - JsInterpreter jsInterpreter, String accountToString) { + protected void persistQuotaTariffUsages(List quotaTariffUsages, Long quotaUsageId) { + for (QuotaTariffUsageVO quotaTariffUsage : quotaTariffUsages) { + quotaTariffUsage.setQuotaUsageId(quotaUsageId); + quotaTariffUsageDao.persistQuotaTariffUsage(quotaTariffUsage); + } + } + + protected Map aggregateQuotaTariffsValues(UsageVO usageRecord, List quotaTariffs, boolean hasAnyQuotaTariffWithActivationRule, + JsInterpreter jsInterpreter, String accountToString) { String usageRecordToString = usageRecord.toString(usageAggregationTimeZone); logger.debug("Validating usage record [{}] for account [{}] against [{}] quota tariffs.", usageRecordToString, accountToString, quotaTariffs.size()); PresetVariables presetVariables = getPresetVariables(hasAnyQuotaTariffWithActivationRule, usageRecord); - BigDecimal aggregatedQuotaTariffsValue = BigDecimal.ZERO; + Map aggregatedQuotaTariffsAndValues = new HashMap<>(); quotaTariffs.sort(Comparator.comparing(QuotaTariffVO::getPosition)); @@ -381,10 +412,9 @@ protected BigDecimal aggregateQuotaTariffsValues(UsageVO usageRecord, List *
    diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index cd87052e878c..23020292027c 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -35,6 +35,7 @@ import javax.inject.Inject; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -783,10 +784,21 @@ protected void loadPresetVariableValueForNetwork(UsageVO usageRecord, Value valu value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); - + value.setResourceCounting(getPresetVariableValueNetworkResourceCounting(networkId)); value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); } + protected ResourceCounting getPresetVariableValueNetworkResourceCounting(Long networkId) { + ResourceCounting resourceCounting = new ResourceCounting(); + List vmInstancesVO = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(networkId, VirtualMachine.Type.User); + int runningVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Running)).count(); + int stoppedVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Stopped)).count(); + + resourceCounting.setRunningVms(runningVms); + resourceCounting.setStoppedVms(stoppedVms); + return resourceCounting; + } + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java new file mode 100644 index 000000000000..75049c3486a3 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + + +import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class ResourceCounting { + + @PresetVariableDefinition(description = "The number of running user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int runningVms; + @PresetVariableDefinition(description = "The number of stopped user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int stoppedVms; + + public int getRunningVms() { + return runningVms; + } + + public void setRunningVms(int runningVms) { + this.runningVms = runningVms; + } + + public int getStoppedVms() { + return stoppedVms; + } + + public void setStoppedVms(int stoppedVms) { + this.stoppedVms = stoppedVms; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index aff2c040e742..286fe5c60fd5 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -84,6 +84,9 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + @PresetVariableDefinition(description = "The amount of resources of the usage type.") + private ResourceCounting resourceCounting; + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; @@ -262,6 +265,14 @@ public void setState(String state) { this.state = state; } + public ResourceCounting getResourceCounting() { + return resourceCounting; + } + + public void setResourceCounting(ResourceCounting resourceCounting) { + this.resourceCounting = resourceCounting; + } + public GenericPresetVariable getNetworkOffering() { return networkOffering; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 0da0d6e53f77..7b725d57c6ef 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -20,11 +20,23 @@ import java.util.HashMap; import java.util.Map; -import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.VpnUser; +import com.cloud.network.rules.LoadBalancer; +import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.security.SecurityGroup; +import com.cloud.network.vpc.Vpc; +import com.cloud.offering.NetworkOffering; +import com.cloud.storage.Snapshot; +import com.cloud.storage.Volume; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.usage.UsageTypes; import org.apache.cloudstack.usage.UsageUnitTypes; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; -import org.apache.commons.lang3.StringUtils; public class QuotaTypes extends UsageTypes { private final Integer quotaType; @@ -32,44 +44,46 @@ public class QuotaTypes extends UsageTypes { private final String quotaUnit; private final String description; private final String discriminator; + private final Class clazz; private final static Map quotaTypeMap; static { final HashMap quotaTypeList = new HashMap(); - quotaTypeList.put(RUNNING_VM, new QuotaTypes(RUNNING_VM, "RUNNING_VM", UsageUnitTypes.COMPUTE_MONTH.toString(), "Running Vm Usage")); - quotaTypeList.put(ALLOCATED_VM, new QuotaTypes(ALLOCATED_VM, "ALLOCATED_VM", UsageUnitTypes.COMPUTE_MONTH.toString(), "Allocated Vm Usage")); - quotaTypeList.put(IP_ADDRESS, new QuotaTypes(IP_ADDRESS, "IP_ADDRESS", UsageUnitTypes.IP_MONTH.toString(), "IP Address Usage")); - quotaTypeList.put(NETWORK_BYTES_SENT, new QuotaTypes(NETWORK_BYTES_SENT, "NETWORK_BYTES_SENT", UsageUnitTypes.GB.toString(), "Network Usage (Bytes Sent)")); - quotaTypeList.put(NETWORK_BYTES_RECEIVED, new QuotaTypes(NETWORK_BYTES_RECEIVED, "NETWORK_BYTES_RECEIVED", UsageUnitTypes.GB.toString(), "Network Usage (Bytes Received)")); - quotaTypeList.put(VOLUME, new QuotaTypes(VOLUME, "VOLUME", UsageUnitTypes.GB_MONTH.toString(), "Volume Usage")); - quotaTypeList.put(TEMPLATE, new QuotaTypes(TEMPLATE, "TEMPLATE", UsageUnitTypes.GB_MONTH.toString(), "Template Usage")); - quotaTypeList.put(ISO, new QuotaTypes(ISO, "ISO", UsageUnitTypes.GB_MONTH.toString(), "ISO Usage")); - quotaTypeList.put(SNAPSHOT, new QuotaTypes(SNAPSHOT, "SNAPSHOT", UsageUnitTypes.GB_MONTH.toString(), "Snapshot Usage")); - quotaTypeList.put(SECURITY_GROUP, new QuotaTypes(SECURITY_GROUP, "SECURITY_GROUP", UsageUnitTypes.POLICY_MONTH.toString(), "Security Group Usage")); - quotaTypeList.put(LOAD_BALANCER_POLICY, new QuotaTypes(LOAD_BALANCER_POLICY, "LOAD_BALANCER_POLICY", UsageUnitTypes.POLICY_MONTH.toString(), "Load Balancer Usage")); - quotaTypeList.put(PORT_FORWARDING_RULE, new QuotaTypes(PORT_FORWARDING_RULE, "PORT_FORWARDING_RULE", UsageUnitTypes.POLICY_MONTH.toString(), "Port Forwarding Usage")); - quotaTypeList.put(NETWORK_OFFERING, new QuotaTypes(NETWORK_OFFERING, "NETWORK_OFFERING", UsageUnitTypes.POLICY_MONTH.toString(), "Network Offering Usage")); - quotaTypeList.put(VPN_USERS, new QuotaTypes(VPN_USERS, "VPN_USERS", UsageUnitTypes.POLICY_MONTH.toString(), "VPN users usage")); - quotaTypeList.put(VM_DISK_IO_READ, new QuotaTypes(VM_DISK_IO_READ, "VM_DISK_IO_READ", UsageUnitTypes.IOPS.toString(), "VM Disk usage(I/O Read)")); - quotaTypeList.put(VM_DISK_IO_WRITE, new QuotaTypes(VM_DISK_IO_WRITE, "VM_DISK_IO_WRITE", UsageUnitTypes.IOPS.toString(), "VM Disk usage(I/O Write)")); - quotaTypeList.put(VM_DISK_BYTES_READ, new QuotaTypes(VM_DISK_BYTES_READ, "VM_DISK_BYTES_READ", UsageUnitTypes.BYTES.toString(), "VM Disk usage(Bytes Read)")); - quotaTypeList.put(VM_DISK_BYTES_WRITE, new QuotaTypes(VM_DISK_BYTES_WRITE, "VM_DISK_BYTES_WRITE", UsageUnitTypes.BYTES.toString(), "VM Disk usage(Bytes Write)")); - quotaTypeList.put(VM_SNAPSHOT, new QuotaTypes(VM_SNAPSHOT, "VM_SNAPSHOT", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot storage usage")); - quotaTypeList.put(VOLUME_SECONDARY, new QuotaTypes(VOLUME_SECONDARY, "VOLUME_SECONDARY", UsageUnitTypes.GB_MONTH.toString(), "Volume secondary storage usage")); - quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage")); - quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage")); - quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage")); - quotaTypeList.put(NETWORK, new QuotaTypes(NETWORK, "NETWORK", UsageUnitTypes.COMPUTE_MONTH.toString(), "Network usage")); - quotaTypeList.put(VPC, new QuotaTypes(VPC, "VPC", UsageUnitTypes.COMPUTE_MONTH.toString(), "VPC usage")); + quotaTypeList.put(RUNNING_VM, new QuotaTypes(RUNNING_VM, "RUNNING_VM", UsageUnitTypes.COMPUTE_MONTH.toString(), "Running Vm Usage", VirtualMachine.class)); + quotaTypeList.put(ALLOCATED_VM, new QuotaTypes(ALLOCATED_VM, "ALLOCATED_VM", UsageUnitTypes.COMPUTE_MONTH.toString(), "Allocated Vm Usage", VirtualMachine.class)); + quotaTypeList.put(IP_ADDRESS, new QuotaTypes(IP_ADDRESS, "IP_ADDRESS", UsageUnitTypes.IP_MONTH.toString(), "IP Address Usage", IpAddress.class)); + quotaTypeList.put(NETWORK_BYTES_SENT, new QuotaTypes(NETWORK_BYTES_SENT, "NETWORK_BYTES_SENT", UsageUnitTypes.GB.toString(), "Network Usage (Bytes Sent)", Network.class)); + quotaTypeList.put(NETWORK_BYTES_RECEIVED, new QuotaTypes(NETWORK_BYTES_RECEIVED, "NETWORK_BYTES_RECEIVED", UsageUnitTypes.GB.toString(), "Network Usage (Bytes Received)", Network.class)); + quotaTypeList.put(VOLUME, new QuotaTypes(VOLUME, "VOLUME", UsageUnitTypes.GB_MONTH.toString(), "Volume Usage", Volume.class)); + quotaTypeList.put(TEMPLATE, new QuotaTypes(TEMPLATE, "TEMPLATE", UsageUnitTypes.GB_MONTH.toString(), "Template Usage", VirtualMachineTemplate.class)); + quotaTypeList.put(ISO, new QuotaTypes(ISO, "ISO", UsageUnitTypes.GB_MONTH.toString(), "ISO Usage", VirtualMachineTemplate.class)); + quotaTypeList.put(SNAPSHOT, new QuotaTypes(SNAPSHOT, "SNAPSHOT", UsageUnitTypes.GB_MONTH.toString(), "Snapshot Usage", Snapshot.class)); + quotaTypeList.put(SECURITY_GROUP, new QuotaTypes(SECURITY_GROUP, "SECURITY_GROUP", UsageUnitTypes.POLICY_MONTH.toString(), "Security Group Usage", SecurityGroup.class)); + quotaTypeList.put(LOAD_BALANCER_POLICY, new QuotaTypes(LOAD_BALANCER_POLICY, "LOAD_BALANCER_POLICY", UsageUnitTypes.POLICY_MONTH.toString(), "Load Balancer Usage", LoadBalancer.class)); + quotaTypeList.put(PORT_FORWARDING_RULE, new QuotaTypes(PORT_FORWARDING_RULE, "PORT_FORWARDING_RULE", UsageUnitTypes.POLICY_MONTH.toString(), "Port Forwarding Usage", PortForwardingRule.class)); + quotaTypeList.put(NETWORK_OFFERING, new QuotaTypes(NETWORK_OFFERING, "NETWORK_OFFERING", UsageUnitTypes.POLICY_MONTH.toString(), "Network Offering Usage", NetworkOffering.class)); + quotaTypeList.put(VPN_USERS, new QuotaTypes(VPN_USERS, "VPN_USERS", UsageUnitTypes.POLICY_MONTH.toString(), "VPN users usage", VpnUser.class)); + quotaTypeList.put(VM_DISK_IO_READ, new QuotaTypes(VM_DISK_IO_READ, "VM_DISK_IO_READ", UsageUnitTypes.IOPS.toString(), "VM Disk usage(I/O Read)", Volume.class)); + quotaTypeList.put(VM_DISK_IO_WRITE, new QuotaTypes(VM_DISK_IO_WRITE, "VM_DISK_IO_WRITE", UsageUnitTypes.IOPS.toString(), "VM Disk usage(I/O Write)", Volume.class)); + quotaTypeList.put(VM_DISK_BYTES_READ, new QuotaTypes(VM_DISK_BYTES_READ, "VM_DISK_BYTES_READ", UsageUnitTypes.BYTES.toString(), "VM Disk usage(Bytes Read)", Volume.class)); + quotaTypeList.put(VM_DISK_BYTES_WRITE, new QuotaTypes(VM_DISK_BYTES_WRITE, "VM_DISK_BYTES_WRITE", UsageUnitTypes.BYTES.toString(), "VM Disk usage(Bytes Write)", Volume.class)); + quotaTypeList.put(VM_SNAPSHOT, new QuotaTypes(VM_SNAPSHOT, "VM_SNAPSHOT", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot storage usage", Snapshot.class)); + quotaTypeList.put(VOLUME_SECONDARY, new QuotaTypes(VOLUME_SECONDARY, "VOLUME_SECONDARY", UsageUnitTypes.GB_MONTH.toString(), "Volume secondary storage usage", Volume.class)); + quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage", Snapshot.class)); + quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage", BackupOffering.class)); + quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage", Bucket.class)); + quotaTypeList.put(NETWORK, new QuotaTypes(NETWORK, "NETWORK", UsageUnitTypes.COMPUTE_MONTH.toString(), "Network usage", Network.class)); + quotaTypeList.put(VPC, new QuotaTypes(VPC, "VPC", UsageUnitTypes.COMPUTE_MONTH.toString(), "VPC usage", Vpc.class)); quotaTypeMap = Collections.unmodifiableMap(quotaTypeList); } - private QuotaTypes(Integer quotaType, String name, String unit, String description) { + private QuotaTypes(Integer quotaType, String name, String unit, String description, Class clazz) { this.quotaType = quotaType; this.description = description; this.quotaName = name; this.quotaUnit = unit; this.discriminator = "None"; + this.clazz = clazz; } public static Map listQuotaTypes() { @@ -104,22 +118,20 @@ static public String getDescription(int quotaType) { return null; } + public Class getClazz() { + return clazz; + } + static public QuotaTypes getQuotaType(int quotaType) { return quotaTypeMap.get(quotaType); } - static public QuotaTypes getQuotaTypeByName(String name) { - if (StringUtils.isBlank(name)) { - throw new CloudRuntimeException("Could not retrieve Quota type by name because the value passed as parameter is null, empty, or blank."); - } - - for (QuotaTypes type : quotaTypeMap.values()) { - if (type.getQuotaName().equals(name)) { - return type; - } + static public Class getClazz(int quotaType) { + QuotaTypes t = quotaTypeMap.get(quotaType); + if (t != null) { + return t.getClazz(); } - - throw new CloudRuntimeException(String.format("Could not find Quota type with name [%s].", name)); + return null; } @Override diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java index c694eaeefbe8..2964746bdf06 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java @@ -28,16 +28,14 @@ public interface QuotaBalanceDao extends GenericDao { QuotaBalanceVO saveQuotaBalance(QuotaBalanceVO qb); - List findCreditBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List findCreditBalances(Long accountId, Long domainId, Date startDate, Date endDate); - QuotaBalanceVO findLastBalanceEntry(Long accountId, Long domainId, Date beforeThis); + QuotaBalanceVO getLastQuotaBalanceEntry(Long accountId, Long domainId, Date beforeThis); QuotaBalanceVO findLaterBalanceEntry(Long accountId, Long domainId, Date afterThis); - List findQuotaBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List listQuotaBalances(Long accountId, Long domainId, Date startDate, Date endDate); - List lastQuotaBalanceVO(Long accountId, Long domainId, Date startDate); - - BigDecimal lastQuotaBalance(Long accountId, Long domainId, Date startDate); + BigDecimal getLastQuotaBalance(Long accountId, Long domainId); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java index 01272d1a6184..21553ebd27b0 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java @@ -18,11 +18,14 @@ import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -32,160 +35,104 @@ import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.db.TransactionStatus; @Component public class QuotaBalanceDaoImpl extends GenericDaoBase implements QuotaBalanceDao { + private static final Logger logger = LogManager.getLogger(QuotaBalanceDaoImpl.class); @Override - public QuotaBalanceVO findLastBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + public QuotaBalanceVO getLastQuotaBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + + if (beforeThis != null) { qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LT, beforeThis); - quotaBalanceEntries = search(qb.create(), filter); - return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; } + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO findLaterBalanceEntry(final Long accountId, final Long domainId, final Date afterThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); - quotaBalanceEntries = search(qb.create(), filter); - return quotaBalanceEntries.size() > 0 ? quotaBalanceEntries.get(0) : null; - } + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO saveQuotaBalance(final QuotaBalanceVO qb) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - return persist(qb); - } - }); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(qb)); } @Override - public List findCreditBalance(final Long accountId, final Long domainId, final Date lastbalancedate, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - if ((lastbalancedate != null) && (beforeThis != null) && lastbalancedate.before(beforeThis)) { - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, lastbalancedate, beforeThis); - return search(qb.create(), filter); - } else { - return new ArrayList(); - } + public List findCreditBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + if (ObjectUtils.anyNull(startDate, endDate) || startDate.after(endDate)) { + return new ArrayList<>(); } - }); - } - @Override - public List findQuotaBalance(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((startDate != null) && (endDate != null) && startDate.before(endDate)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } else { - return Collections. emptyList(); - } - quotaUsageRecords = listBy(qb.create()); - if (quotaUsageRecords.size() == 0) { - quotaUsageRecords.addAll(lastQuotaBalanceVO(accountId, domainId, startDate)); - } - return quotaUsageRecords; + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } + return search(qb.create(), filter); }); - } @Override - public List lastQuotaBalanceVO(final Long accountId, final Long domainId, final Date pivotDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - List trimmedRecords = new ArrayList(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 100L); - // ASSUMPTION there will be less than 100 continuous credit - // transactions - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((pivotDate != null)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, pivotDate); - } - quotaUsageRecords = search(qb.create(), filter); - - // get records before startDate to find start balance - for (QuotaBalanceVO entry : quotaUsageRecords) { - if (logger.isDebugEnabled()) { - logger.debug("FindQuotaBalance Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - trimmedRecords.add(entry); - } else { - trimmedRecords.add(entry); - break; // add only consecutive credit entries and last balance entry - } - } - return trimmedRecords; + public List listQuotaBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + if (startDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GTEQ, startDate); + } + if (endDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, endDate); } + + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true); + return listBy(qb.create(), filter); }); } @Override - public BigDecimal lastQuotaBalance(final Long accountId, final Long domainId, Date startDate) { - List quotaBalance = lastQuotaBalanceVO(accountId, domainId, startDate); - BigDecimal finalBalance = new BigDecimal(0); - if (quotaBalance.isEmpty()) { - logger.info("There are no balance entries on or before the requested date."); - return finalBalance; + public BigDecimal getLastQuotaBalance(final Long accountId, final Long domainId) { + QuotaBalanceVO quotaBalance = getLastQuotaBalanceEntry(accountId, domainId, null); + BigDecimal finalBalance = BigDecimal.ZERO; + Date startDate = DateUtils.addDays(new Date(), -1); + if (quotaBalance == null) { + logger.info("There are no balance entries for account [{}] and domain [{}]. Considering only new added credits.", accountId, domainId); + } else { + finalBalance = quotaBalance.getCreditBalance(); + startDate = quotaBalance.getUpdatedOn(); } - for (QuotaBalanceVO entry : quotaBalance) { - if (logger.isDebugEnabled()) { - logger.debug("lastQuotaBalance Entry=" + entry); - } - finalBalance = finalBalance.add(entry.getCreditBalance()); + + List credits = findCreditBalances(accountId, domainId, startDate, new Date()); + + for (QuotaBalanceVO credit : credits) { + finalBalance = finalBalance.add(credit.getCreditBalance()); } + return finalBalance; } + private QueryBuilder getQuotaBalanceQueryBuilder(Long accountId, Long domainId) { + QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); + qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); + qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); + return qb; + } + } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java index 126fa11413f7..ead70ae35012 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java @@ -26,6 +26,6 @@ public interface QuotaUsageJoinDao extends GenericDao { - List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); + List findQuotaUsage(Long accountId, List domainIds, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java index b98ea2b3a5d2..e69b1a05ba4d 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java @@ -61,7 +61,7 @@ public void init() { private void prepareQuotaUsageSearchBuilder(SearchBuilder searchBuilder) { searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); - searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainIds", searchBuilder.entity().getDomainId(), SearchCriteria.Op.IN); searchBuilder.and("usageType", searchBuilder.entity().getUsageType(), SearchCriteria.Op.EQ); searchBuilder.and("resourceId", searchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); searchBuilder.and("networkId", searchBuilder.entity().getNetworkId(), SearchCriteria.Op.EQ); @@ -71,11 +71,13 @@ private void prepareQuotaUsageSearchBuilder(SearchBuilder sear } @Override - public List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { + public List findQuotaUsage(Long accountId, List domainIds, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { SearchCriteria sc = tariffId == null ? searchQuotaUsages.create() : searchQuotaUsagesJoinTariffUsages.create(); sc.setParametersIfNotNull("accountId", accountId); - sc.setParametersIfNotNull("domainId", domainId); + if (domainIds != null) { + sc.setParameters("domainIds", domainIds.toArray()); + } sc.setParametersIfNotNull("usageType", usageType); sc.setParametersIfNotNull("resourceId", resourceId); sc.setParametersIfNotNull("networkId", networkId); diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java index a33faa054de4..1e08e7d7fc00 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -33,7 +34,9 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaTariffDao; +import org.apache.cloudstack.quota.dao.QuotaTariffUsageDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.apache.cloudstack.usage.UsageUnitTypes; @@ -89,6 +92,9 @@ public class QuotaManagerImplTest { @Mock PresetVariables presetVariablesMock; + @Mock + QuotaTariffUsageDao quotaTariffUsageDaoMock; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @Test @@ -484,47 +490,49 @@ public void getPresetVariablesTestHasTariffsWithActivationRuleReturnPresetVariab } @Test - public void aggregateQuotaTariffsValuesTestTariffsWereNotInPeriodToBeAppliedReturnZero() { + public void aggregateQuotaTariffsValuesTestTariffsWereNotInPeriodToBeAppliedReturnEmptyMap() { List tariffs = createTariffList(); Mockito.doReturn(false).when(quotaManagerImplSpy).isQuotaTariffInPeriodToBeApplied(Mockito.any(), Mockito.any(), Mockito.anyString()); - BigDecimal result = quotaManagerImplSpy.aggregateQuotaTariffsValues(usageVoMock, tariffs, false, jsInterpreterMock, ""); + Map result = quotaManagerImplSpy.aggregateQuotaTariffsValues(usageVoMock, tariffs, false, jsInterpreterMock, ""); - Assert.assertEquals(BigDecimal.ZERO, result); + Assert.assertTrue(result.isEmpty()); } @Test - public void aggregateQuotaTariffsValuesTestTariffsIsEmptyReturnZero() { - BigDecimal result = quotaManagerImplSpy.aggregateQuotaTariffsValues(usageVoMock, new ArrayList<>(), false, jsInterpreterMock, ""); + public void aggregateQuotaTariffsValuesTestTariffsIsEmptyReturnEmptyMap() { + Map result = quotaManagerImplSpy.aggregateQuotaTariffsValues(usageVoMock, new ArrayList<>(), false, jsInterpreterMock, ""); - Assert.assertEquals(BigDecimal.ZERO, result); + Assert.assertTrue(result.isEmpty()); } @Test - public void aggregateQuotaTariffsValuesTestTariffsAreInPeriodToBeAppliedReturnAggregation() { + public void aggregateQuotaTariffsValuesTestTariffsAreInPeriodToBeAppliedReturnTariffsWithTheirValues() { List tariffs = createTariffList(); Mockito.doReturn(true, false, true).when(quotaManagerImplSpy).isQuotaTariffInPeriodToBeApplied(Mockito.any(), Mockito.any(), Mockito.anyString()); Mockito.doReturn(BigDecimal.TEN).when(quotaManagerImplSpy).getQuotaTariffValueToBeApplied(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); - BigDecimal result = quotaManagerImplSpy.aggregateQuotaTariffsValues(usageVoMock, tariffs, false, jsInterpreterMock, ""); + Map result = quotaManagerImplSpy.aggregateQuotaTariffsValues(usageVoMock, tariffs, false, jsInterpreterMock, ""); - Assert.assertEquals(BigDecimal.TEN.multiply(new BigDecimal(2)), result); + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.values().stream().allMatch(value -> value.equals(BigDecimal.TEN))); } @Test public void persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsagesTestReturnOnlyPersistedQuotaUsageVo() { - List> listPair = new ArrayList<>(); + Map>> mapUsageQuotaUsage = new LinkedHashMap<>(); QuotaUsageVO quotaUsageVoMock1 = Mockito.mock(QuotaUsageVO.class); QuotaUsageVO quotaUsageVoMock2 = Mockito.mock(QuotaUsageVO.class); - listPair.add(new Pair<>(new UsageVO(), quotaUsageVoMock1)); - listPair.add(new Pair<>(new UsageVO(), null)); - listPair.add(new Pair<>(new UsageVO(), quotaUsageVoMock2)); + mapUsageQuotaUsage.put(new UsageVO(), new Pair<>(quotaUsageVoMock1, new ArrayList<>())); + mapUsageQuotaUsage.put(new UsageVO(), null); + mapUsageQuotaUsage.put(new UsageVO(), new Pair<>(quotaUsageVoMock2, new ArrayList<>())); Mockito.doReturn(null).when(usageDaoMock).persistUsage(Mockito.any()); Mockito.doReturn(null).when(quotaUsageDaoMock).persistQuotaUsage(Mockito.any()); + Mockito.doNothing().when(quotaManagerImplSpy).persistQuotaTariffUsages(Mockito.any(), Mockito.any()); - List result = quotaManagerImplSpy.persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsages(listPair); + List result = quotaManagerImplSpy.persistUsagesAndQuotaUsagesAndRetrievePersistedQuotaUsages(mapUsageQuotaUsage); Assert.assertEquals(2, result.size()); Assert.assertEquals(quotaUsageVoMock1, result.get(0)); @@ -551,4 +559,41 @@ private static List createLastAppliedTariffsPresetVariableList(int numbe return lastTariffs; } + @Test + public void createQuotaTariffUsageTestTariffValueIsNotZeroReturnDetailedVo() { + Mockito.doReturn(1).when(usageVoMock).getUsageType(); + Mockito.doReturn(BigDecimal.TEN).when(quotaManagerImplSpy).getUsageValueAccordingToUsageUnitType(Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.doReturn(1L).when(quotaTariffVoMock).getId(); + + QuotaTariffUsageVO result = quotaManagerImplSpy.createQuotaTariffUsage(usageVoMock, quotaTariffVoMock, BigDecimal.TEN); + + Assert.assertEquals(1L, result.getTariffId().longValue()); + Assert.assertEquals(BigDecimal.TEN, result.getQuotaUsed()); + } + + @Test + public void persistQuotaTariffUsagesTestZeroQuotaTariffUsagesZeroPersisted() { + List quotaTariffUsages = new ArrayList<>(); + + quotaManagerImplSpy.persistQuotaTariffUsages(quotaTariffUsages, 1L); + + Mockito.verify(quotaTariffUsageDaoMock, Mockito.never()).persistQuotaTariffUsage(Mockito.any()); + } + + @Test + public void persistQuotaTariffUsagesTestTwoQuotaUsageDetailsTwoPersisted() { + List quotaUsageDetails = new ArrayList<>(); + QuotaTariffUsageVO quotaUsageDetailVoMock1 = Mockito.mock(QuotaTariffUsageVO.class); + QuotaTariffUsageVO quotaUsageDetailVoMock2 = Mockito.mock(QuotaTariffUsageVO.class); + + quotaUsageDetails.add(quotaUsageDetailVoMock1); + quotaUsageDetails.add(quotaUsageDetailVoMock2); + + Mockito.doNothing().when(quotaTariffUsageDaoMock).persistQuotaTariffUsage(Mockito.any()); + + quotaManagerImplSpy.persistQuotaTariffUsages(quotaUsageDetails, 1L); + + Mockito.verify(quotaTariffUsageDaoMock, Mockito.times(2)).persistQuotaTariffUsage(Mockito.any()); + } + } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index fb093a456790..bcdbc3b46cec 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,11 +31,13 @@ import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -238,6 +240,8 @@ private Value getValueForTests() { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setState(Network.State.Implemented.toString()); + value.setResourceCounting(getResourceCountingForTests()); value.setNetworkOffering(getNetworkOfferingForTests()); value.setVpcOffering(getVpcOfferingForTests()); return value; @@ -276,6 +280,13 @@ private Configuration getConfigurationForTests() { return configuration; } + private ResourceCounting getResourceCountingForTests() { + ResourceCounting resourceCounting = new ResourceCounting(); + resourceCounting.setRunningVms(1); + resourceCounting.setStoppedVms(1); + return resourceCounting; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -1343,8 +1354,8 @@ public void loadPresetVariableValueForNetworkTestRecordIsNetworkSetFields() { Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getResourceCounting()).when(presetVariableHelperSpy).getPresetVariableValueNetworkResourceCounting(Mockito.anyLong()); Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); - Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); Value result = new Value(); @@ -1352,7 +1363,27 @@ public void loadPresetVariableValueForNetworkTestRecordIsNetworkSetFields() { assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getState(), result.getState()); - Assert.assertEquals(expected.getNetworkOffering(), result.getNetworkOffering()); + Assert.assertEquals(expected.getResourceCounting(), result.getResourceCounting()); + } + + @Test + public void getPresetVariableValueNetworkResourceCountingTestSetValueAndReturnObject() { + VMInstanceVO vmInstanceVoMock1 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock1.setState(VirtualMachine.State.Stopped); + + VMInstanceVO vmInstanceVoMock2 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock2.setState(VirtualMachine.State.Running); + + Mockito.doReturn(List.of(vmInstanceVoMock1, vmInstanceVoMock2)).when(vmInstanceDaoMock).listNonRemovedVmsByTypeAndNetwork(Mockito.anyLong(), Mockito.any()); + + mockMethodValidateIfObjectIsNull(); + + ResourceCounting expected = getResourceCountingForTests(); + + ResourceCounting result = presetVariableHelperSpy.getPresetVariableValueNetworkResourceCounting(1L); + + Assert.assertEquals(expected.getRunningVms(), result.getRunningVms()); + Assert.assertEquals(expected.getStoppedVms(), result.getStoppedVms()); } @Test @@ -1362,7 +1393,7 @@ public void loadPresetVariableValueForVpcTestRecordIsNotAVpcDoNothing() { presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); }); - Mockito.verifyNoInteractions(networkDaoMock); + Mockito.verifyNoInteractions(vpcDaoMock); } @Test diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java new file mode 100644 index 000000000000..afdd88f8d1da --- /dev/null +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java @@ -0,0 +1,91 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; + +import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class QuotaBalanceDaoImplTest { + QuotaBalanceDaoImpl quotaBalanceDaoImplSpy = Mockito.spy(QuotaBalanceDaoImpl.class); + + @Mock + QuotaBalanceVO quotaBalanceVoMock; + + @Test + public void getLastQuotaBalanceTestLastEntryIsNullAndNoCreditsReturnsZero() { + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(1L, 2L); + + Assert.assertEquals(BigDecimal.ZERO, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryAndNoCredits() { + BigDecimal expected = BigDecimal.valueOf(-1542.46); + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryPlusCredits() { + BigDecimal balance = BigDecimal.valueOf(-1542.46); + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = balance.add(credit1).add(credit2); + + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(balance, credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryIsNullPlusCredits() { + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = credit1.add(credit2); + + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } +} diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 3dee161bf274..3ba2e4d5789e 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -117,7 +117,7 @@ Requires: ipset Requires: perl Requires: rsync Requires: cifs-utils -Requires: edk2-ovmf +Requires: (edk2-ovmf or qemu-ovmf-x86_64) Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) Requires: (qemu-img or qemu-tools) @@ -127,6 +127,8 @@ Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat +Requires: python3-libnbd +Requires: socat Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/packaging/suse15 b/packaging/suse15 deleted file mode 120000 index 4dad90d45e0c..000000000000 --- a/packaging/suse15 +++ /dev/null @@ -1 +0,0 @@ -el8 \ No newline at end of file diff --git a/packaging/suse15/cloud-ipallocator.rc b/packaging/suse15/cloud-ipallocator.rc new file mode 120000 index 000000000000..647598e6dc41 --- /dev/null +++ b/packaging/suse15/cloud-ipallocator.rc @@ -0,0 +1 @@ +../el8/cloud-ipallocator.rc \ No newline at end of file diff --git a/packaging/suse15/cloud.limits b/packaging/suse15/cloud.limits new file mode 120000 index 000000000000..37be77e3acf2 --- /dev/null +++ b/packaging/suse15/cloud.limits @@ -0,0 +1 @@ +../el8/cloud.limits \ No newline at end of file diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec new file mode 100644 index 000000000000..cdfc5a72a34e --- /dev/null +++ b/packaging/suse15/cloud.spec @@ -0,0 +1,750 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +%define __os_install_post %{nil} +%global debug_package %{nil} +%global __requires_exclude libc\\.so\\..*|libc\\.so\\.6\\(GLIBC_.*\\) +%define _binaries_in_noarch_packages_terminate_build 0 + +# DISABLE the post-percentinstall java repacking and line number stripping +# we need to find a way to just disable the java repacking and line number stripping, but not the autodeps + +Name: cloudstack +Summary: CloudStack IaaS Platform +#http://fedoraproject.org/wiki/PackageNamingGuidelines#Pre-Release_packages +%define _maventag %{_fullver} +Release: %{_rel} + +Version: %{_ver} +License: ASL 2.0 +Vendor: Apache CloudStack +Packager: Apache CloudStack +Group: System Environment/Libraries +# FIXME do groups for every single one of the subpackages +Source0: %{name}-%{_maventag}.tgz +BuildRoot: %{_tmppath}/%{name}-%{_maventag}-%{release}-build +BuildArch: noarch + +BuildRequires: (java-11-openjdk-devel or java-17-openjdk-devel or java-21-openjdk-devel) +#BuildRequires: ws-commons-util +BuildRequires: jpackage-utils +BuildRequires: gcc +BuildRequires: glibc-devel +BuildRequires: /usr/bin/mkisofs +BuildRequires: python3-setuptools +BuildRequires: wget +BuildRequires: nodejs + +%description +CloudStack is a highly-scalable elastic, open source, +intelligent IaaS cloud implementation. + +%package management +Summary: CloudStack management server UI +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Requires: python3 +Requires: bash +Requires: gawk +Requires: which +Requires: file +Requires: tar +Requires: bzip2 +Requires: gzip +Requires: unzip +Requires: (/sbin/mount.nfs or /usr/sbin/mount.nfs) +Requires: (openssh-clients or openssh) +Requires: (nfs-utils or nfs-client) +Requires: iproute +Requires: wget +Requires: (mysql or mariadb or mysql8.4) +Requires: sudo +Requires: /sbin/service +Requires: /sbin/chkconfig +Requires: /usr/bin/ssh-keygen +Requires: (genisoimage or mkisofs or xorrisofs) +Requires: ipmitool +Requires: %{name}-common = %{_ver} +Requires: (iptables-services or iptables) +Requires: rng-tools +Requires: (qemu-img or qemu-tools) +Requires: python3-pip +Requires: python3-six +Requires: python3-protobuf +Requires: python3-setuptools +Requires: (libgcrypt > 1.8.3 or libgcrypt20) +Group: System Environment/Libraries +%description management +The CloudStack management server is the central point of coordination, +management, and intelligence in CloudStack. + +%package common +Summary: Apache CloudStack common files and scripts +Requires: python3 +Group: System Environment/Libraries +%description common +The Apache CloudStack files shared between agent and management server +%global __requires_exclude libc\\.so\\..*|libc\\.so\\.6\\(GLIBC_.*\\)|^(libuuid\\.so\\.1|/usr/bin/python)$ + +%package agent +Summary: CloudStack Agent for KVM hypervisors +Requires: (openssh-clients or openssh) +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Requires: %{name}-common = %{_ver} +Requires: libvirt +Requires: libvirt-daemon-driver-storage-rbd +Requires: ebtables +Requires: iptables +Requires: ethtool +Requires: (net-tools or net-tools-deprecated) +Requires: iproute +Requires: ipset +Requires: perl +Requires: rsync +Requires: cifs-utils +Requires: (edk2-ovmf or qemu-ovmf-x86_64) +Requires: swtpm +Requires: (python3-libvirt or python3-libvirt-python) +Requires: (qemu-img or qemu-tools) +Requires: qemu-kvm +Requires: cryptsetup +Requires: rng-tools +Requires: (libgcrypt > 1.8.3 or libgcrypt20) +Requires: (selinux-tools if selinux-tools) +Requires: sysstat +Provides: cloud-agent +Group: System Environment/Libraries +%description agent +The CloudStack agent for KVM hypervisors + +%package baremetal-agent +Summary: CloudStack baremetal agent +Requires: tftp-server +Requires: xinetd +Requires: syslinux +Requires: chkconfig +Requires: dhcp +Requires: httpd +Group: System Environment/Libraries +%description baremetal-agent +The CloudStack baremetal agent + +%package usage +Summary: CloudStack Usage calculation server +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Group: System Environment/Libraries +%description usage +The CloudStack usage calculation service + +%package ui +Summary: CloudStack UI +Group: System Environment/Libraries +%description ui +The CloudStack UI + +%package marvin +Summary: Apache CloudStack Marvin library +Requires: python3-pip +Requires: gcc +Requires: python3-devel +Requires: libffi-devel +Requires: openssl-devel +Group: System Environment/Libraries +%description marvin +Apache CloudStack Marvin library + +%package integration-tests +Summary: Apache CloudStack Marvin integration tests +Requires: %{name}-marvin = %{_ver} +Group: System Environment/Libraries +%description integration-tests +Apache CloudStack Marvin integration tests + +%if "%{_ossnoss}" == "noredist" +%package mysql-ha +Summary: Apache CloudStack Balancing Strategy for MySQL +Group: System Environmnet/Libraries +%description mysql-ha +Apache CloudStack Balancing Strategy for MySQL + +%endif + +%prep +echo Doing CloudStack build + +%setup -q -n %{name}-%{_maventag} + +%build + +cp packaging/suse15/replace.properties build/replace.properties +echo VERSION=%{_maventag} >> build/replace.properties +echo PACKAGE=%{name} >> build/replace.properties +touch build/gitrev.txt +echo $(git rev-parse HEAD) > build/gitrev.txt + +if [ "%{_ossnoss}" == "NOREDIST" -o "%{_ossnoss}" == "noredist" ] ; then + echo "Adding noredist flag to the maven build" + FLAGS="$FLAGS -Dnoredist" +fi + +if [ "%{_sim}" == "SIMULATOR" -o "%{_sim}" == "simulator" ] ; then + echo "Adding simulator flag to the maven build" + FLAGS="$FLAGS -Dsimulator" +fi + +if [ \"%{_temp}\" != "" ]; then + echo "Adding flags to package requested templates" + FLAGS="$FLAGS `rpm --eval %{?_temp}`" +fi + +mvn -Psystemvm,developer $FLAGS clean package +cd ui && npm install && npm run build && cd .. + +%install +[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT} +# Common directories +mkdir -p ${RPM_BUILD_ROOT}%{_bindir} +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/ipallocator +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_initrddir} +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/default +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d + +# Common +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site +mkdir -p ${RPM_BUILD_ROOT}/usr/bin +cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +install -D systemvm/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/ +install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloud_utils.py +cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/ +python3 -m py_compile ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloud_utils.py +python3 -m compileall ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloudutils +cp build/gitrev.txt ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +cp packaging/suse15/cloudstack-sccs ${RPM_BUILD_ROOT}/usr/bin + +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco +cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco + +# Management +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/run +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel + +# Setup Jetty +ln -sf /etc/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/conf +ln -sf /var/log/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/logs + +install -D client/target/utilities/bin/cloud-migrate-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-migrate-databases +install -D client/target/utilities/bin/cloud-set-guest-password ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-password +install -D client/target/utilities/bin/cloud-set-guest-sshkey ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-sshkey +install -D client/target/utilities/bin/cloud-setup-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-databases +install -D client/target/utilities/bin/cloud-setup-encryption ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-encryption +install -D client/target/utilities/bin/cloud-setup-management ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-management +install -D client/target/utilities/bin/cloud-setup-baremetal ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-baremetal +install -D client/target/utilities/bin/cloud-sysvmadm ${RPM_BUILD_ROOT}%{_bindir}/%{name}-sysvmadm +install -D client/target/utilities/bin/cloud-update-xenserver-licenses ${RPM_BUILD_ROOT}%{_bindir}/%{name}-update-xenserver-licenses +# Bundle cmk in cloudstack-management +wget https://github.com/apache/cloudstack-cloudmonkey/releases/latest/download/cmk.linux.x86-64 -O ${RPM_BUILD_ROOT}%{_bindir}/cmk +chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk + +cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf +cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ +cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp +cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/ +cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/ +rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/config.json +ln -sf /etc/%{name}/management/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/config.json +mv ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/cloudstack-%{_maventag}.jar +cp client/target/lib/*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/ + +# Don't package the scripts in the management webapp +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/scripts +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/vms + +for name in db.properties server.properties log4j-cloud.xml environment.properties java.security.ciphers +do + cp client/target/conf/$name ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/$name +done + +ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j2.xml + +install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py +install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar + +install -D packaging/suse15/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator +install -D packaging/suse15/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud +install -D packaging/suse15/filelimit.conf ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d +install -D packaging/systemd/cloudstack-management.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-management.service +install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-management +install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management +touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid +#install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina +install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management + +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml + +# SystemVM template +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm +cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm/sha512sum.txt + +# Sample Extensions +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/extensions +cp -r extensions/* ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/extensions +ln -sf %{_sysconfdir}/%{name}/extensions ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/extensions + +# UI +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/ui +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ +cp -r client/target/classes/META-INF/webapp/WEB-INF ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui +cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/ui/ +cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ +rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json +ln -sf /etc/%{name}/ui/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json + +# Package mysql-connector-python (bundled to avoid dependency on external community repo) +# Version 8.0.31 is the last version supporting Python 3.6 (EL8) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Version 8.3.0 supports Python 3.8 to 3.12 (EL9, EL10) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/53/ed/26a4b8cacb8852c6fd97d2d58a7f2591c41989807ea82bd8d9725a4e6937/mysql_connector_python-8.3.0-py2.py3-none-any.whl + +chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent + +# KVM Agent +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/plugins +install -D packaging/systemd/cloudstack-agent.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-agent.service +install -D packaging/systemd/cloudstack-rolling-maintenance@.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-rolling-maintenance@.service +install -D packaging/systemd/cloudstack-agent.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-agent +install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/agent.properties +install -D agent/target/transformed/uefi.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/uefi.properties +install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties +install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml +install -D agent/target/transformed/cloud-setup-agent ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-agent +install -D agent/target/transformed/cloudstack-agent-upgrade ${RPM_BUILD_ROOT}%{_bindir}/%{name}-agent-upgrade +install -D agent/target/transformed/cloud-guest-tool ${RPM_BUILD_ROOT}%{_bindir}/%{name}-guest-tool +install -D agent/target/transformed/libvirtqemuhook ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook +install -D agent/target/transformed/rolling-maintenance ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/rolling-maintenance +install -D agent/target/transformed/cloud-ssh ${RPM_BUILD_ROOT}%{_bindir}/%{name}-ssh +install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d/%{name}-agent-profile.sh +install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent +install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar +cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +cp plugins/storage/volume/storpool/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +cp plugins/storage/volume/linstor/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib + +# Usage server +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib +install -D usage/target/cloud-usage-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/cloud-usage-%{_maventag}.jar +install -D usage/target/transformed/db.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/db.properties +install -D usage/target/transformed/log4j-cloud_usage.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/log4j-cloud.xml +cp usage/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ +cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ +install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service +install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/ +install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage + +# Marvin +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin +cp tools/marvin/dist/Marvin-*.tar.gz ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin/ + +# integration-tests +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests +cp -r test/integration/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests/ + +# MYSQL HA +if [ "x%{_ossnoss}" == "xnoredist" ] ; then + mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib + cp -r plugins/database/mysql-ha/target/cloud-plugin-database-mysqlha-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib +fi + +#License files from whisker +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-ui-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-ui-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE + +%clean +[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT} + +%posttrans common + +unalias cp +python_dir=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +if [ ! -z $python_dir ];then + cp -f -r /usr/share/cloudstack-common/python-site/* $python_dir/ +fi + +%preun management +/usr/bin/systemctl stop cloudstack-management || true +/usr/bin/systemctl disable cloudstack-management || true + +%pre management +id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -U -c "CloudStack unprivileged user" \ + -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud || true + +rm -rf %{_localstatedir}/cache/cloudstack + +# in case of upgrade to 4.9+ copy commands.properties if not exists in /etc/cloudstack/management/ +if [ "$1" == "2" ] ; then + if [ -f "%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties" ] && [ ! -f "%{_sysconfdir}/%{name}/management/commands.properties" ] ; then + cp -p %{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties %{_sysconfdir}/%{name}/management/commands.properties + fi +fi + +# Remove old tomcat symlinks and env config file +if [ -L "%{_datadir}/%{name}-management/lib" ] +then + rm -f %{_datadir}/%{name}-management/bin + rm -f %{_datadir}/%{name}-management/lib + rm -f %{_datadir}/%{name}-management/temp + rm -f %{_datadir}/%{name}-management/work + rm -f %{_sysconfdir}/default/%{name}-management +fi + +%post management +# Install mysql-connector-python wheel +# Detect Python version to install compatible wheel +if python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 7) else 1)'; then + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.3.0-py2.py3-none-any.whl +else + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +fi + +/usr/bin/systemctl enable cloudstack-management > /dev/null 2>&1 || true +/usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true + +grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "db.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "db.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" + +# Update DB properties having master and slave(s), with source and replica(s) respectively (for inclusiveness) +grep -s -q "^db.cloud.slaves=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.slaves=/db.cloud.replicas=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.cloud.secondsBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.secondsBeforeRetryMaster=/db.cloud.secondsBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.cloud.queriesBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.queriesBeforeRetryMaster=/db.cloud.queriesBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.slaves=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.slaves=/db.usage.replicas=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.secondsBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.secondsBeforeRetryMaster=/db.usage.secondsBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.queriesBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.queriesBeforeRetryMaster=/db.usage.queriesBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" + +if [ ! -f %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/vhd-util ] ; then + echo Please download vhd-util from http://download.cloudstack.org/tools/vhd-util and put it in + echo %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/ +fi + +if [ -f %{_sysconfdir}/sysconfig/%{name}-management ] ; then + rm -f %{_sysconfdir}/sysconfig/%{name}-management +fi + +chown -R cloud:cloud /var/log/cloudstack/management +chown -R cloud:cloud /usr/share/cloudstack-management/templates +find /usr/share/cloudstack-management/templates -type d -exec chmod 0770 {} \; + +systemctl daemon-reload + +%posttrans management +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text management +fi + +%preun agent +/sbin/service cloudstack-agent stop || true +if [ "$1" == "0" ] ; then + /sbin/chkconfig --del cloudstack-agent > /dev/null 2>&1 || true +fi + +%pre agent + +# save old configs if they exist (for upgrade). Otherwise we may lose them +# when the old packages are erased. There are a lot of properties files here. +if [ -d "%{_sysconfdir}/cloud" ] ; then + mv %{_sysconfdir}/cloud %{_sysconfdir}/cloud.rpmsave +fi + +%posttrans agent + +if [ "$1" == "2" ] ; then + echo "Running %{_bindir}/%{name}-agent-upgrade to update bridge name for upgrade from CloudStack 4.0.x (and before) to CloudStack 4.1 (and later)" + %{_bindir}/%{name}-agent-upgrade +fi +if [ ! -d %{_sysconfdir}/libvirt/hooks ] ; then + mkdir %{_sysconfdir}/libvirt/hooks +fi +cp -a ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook %{_sysconfdir}/libvirt/hooks/qemu +mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp +/usr/bin/systemctl restart libvirtd +/usr/bin/systemctl enable cloudstack-agent > /dev/null 2>&1 || true +/usr/bin/systemctl enable cloudstack-rolling-maintenance@p > /dev/null 2>&1 || true +/usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true + +# if saved agent.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/agent.properties %{_sysconfdir}/%{name}/agent/agent.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave +fi + +# if saved uefi.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/uefi.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/uefi.properties %{_sysconfdir}/%{name}/agent/uefi.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties.rpmsave +fi + +systemctl daemon-reload + +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text agent +fi + +%pre usage +id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -U -c "CloudStack unprivileged user" \ + -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true + +%preun usage +/sbin/service cloudstack-usage stop || true +if [ "$1" == "0" ] ; then + /sbin/chkconfig --del cloudstack-usage > /dev/null 2>&1 || true +fi + +%post usage +if [ -f "%{_sysconfdir}/%{name}/management/db.properties" ]; then + echo "Replacing usage server's db.properties with a link to the management server's db.properties" + rm -f %{_sysconfdir}/%{name}/usage/db.properties + ln -s %{_sysconfdir}/%{name}/management/db.properties %{_sysconfdir}/%{name}/usage/db.properties + /usr/bin/systemctl enable cloudstack-usage > /dev/null 2>&1 || true +fi + +if [ -f "%{_sysconfdir}/%{name}/management/key" ]; then + echo "Replacing usage server's key with a link to the management server's key" + rm -f %{_sysconfdir}/%{name}/usage/key + ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key +fi + +if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then + ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key +fi + +mkdir -p /usr/local/libexec +if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then + echo 1 > /usr/local/libexec/sanity-check-last-id +fi +chown cloud:cloud /usr/local/libexec/sanity-check-last-id + +%posttrans usage +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text usage +fi + +%post marvin +pip3 install --upgrade https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz + +#No default permission as the permission setup is complex +%files management +%defattr(-,root,root,-) +%dir %{_datadir}/%{name}-management +%dir %attr(0770,root,cloud) %{_localstatedir}/%{name}/mnt +%dir %attr(0770,cloud,cloud) %{_localstatedir}/%{name}/management +%dir %attr(0770,root,cloud) %{_localstatedir}/cache/%{name}/management +%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/management +%config(noreplace) %{_sysconfdir}/default/%{name}-management +%config(noreplace) %{_sysconfdir}/sudoers.d/%{name}-management +%config(noreplace) %{_sysconfdir}/security/limits.d/cloud +%config(noreplace) %{_sysconfdir}/systemd/system/%{name}-management.service.d +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/db.properties +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/server.properties +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/config.json +%config(noreplace) %{_sysconfdir}/%{name}/management/log4j-cloud.xml +%config(noreplace) %{_sysconfdir}/%{name}/management/log4j2.xml +%config(noreplace) %{_sysconfdir}/%{name}/management/environment.properties +%config(noreplace) %{_sysconfdir}/%{name}/management/java.security.ciphers +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management +%attr(0644,root,root) %{_unitdir}/%{name}-management.service +%attr(0755,cloud,cloud) %{_localstatedir}/run/%{name}-management.pid +%attr(0755,root,root) %{_bindir}/%{name}-setup-management +%attr(0755,root,root) %{_bindir}/%{name}-update-xenserver-licenses +%{_datadir}/%{name}-management/conf +%{_datadir}/%{name}-management/lib/*.jar +%{_datadir}/%{name}-management/logs +%{_datadir}/%{name}-management/templates +%{_datadir}/%{name}-management/extensions +%attr(0755,root,root) %{_bindir}/%{name}-setup-databases +%attr(0755,root,root) %{_bindir}/%{name}-migrate-databases +%attr(0755,root,root) %{_bindir}/%{name}-set-guest-password +%attr(0755,root,root) %{_bindir}/%{name}-set-guest-sshkey +%attr(0755,root,root) %{_bindir}/%{name}-sysvmadm +%attr(0755,root,root) %{_bindir}/%{name}-setup-encryption +%attr(0755,root,root) %{_bindir}/cmk +%{_datadir}/%{name}-management/cks/conf/*.yml +%{_datadir}/%{name}-management/setup/*.sql +%{_datadir}/%{name}-management/setup/*.sh +%{_datadir}/%{name}-management/setup/server-setup.xml +%{_datadir}/%{name}-management/webapp/* +%dir %attr(0770, cloud, cloud) %{_datadir}/%{name}-management/templates +%dir %attr(0770, cloud, cloud) %{_datadir}/%{name}-management/templates/systemvm +%attr(0644, cloud, cloud) %{_datadir}/%{name}-management/templates/systemvm/* +%attr(0755,root,root) %{_bindir}/%{name}-external-ipallocator.py +%attr(0755,root,root) %{_initrddir}/%{name}-ipallocator +%dir %attr(0770,root,root) %{_localstatedir}/log/%{name}/ipallocator +%{_defaultdocdir}/%{name}-management-%{version}/LICENSE +%{_defaultdocdir}/%{name}-management-%{version}/NOTICE +%{_datadir}/%{name}-management/setup/wheel/*.whl +%dir %attr(0755,cloud,cloud) %{_sysconfdir}/%{name}/extensions +%attr(0755,cloud,cloud) %{_sysconfdir}/%{name}/extensions/* + +%files agent +%attr(0755,root,root) %{_bindir}/%{name}-setup-agent +%attr(0755,root,root) %{_bindir}/%{name}-agent-upgrade +%attr(0755,root,root) %{_bindir}/%{name}-guest-tool +%attr(0755,root,root) %{_bindir}/%{name}-ssh +%attr(0644,root,root) %{_unitdir}/%{name}-agent.service +%attr(0644,root,root) %{_unitdir}/%{name}-rolling-maintenance@.service +%config(noreplace) %{_sysconfdir}/default/%{name}-agent +%attr(0644,root,root) %{_sysconfdir}/profile.d/%{name}-agent-profile.sh +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-agent +%attr(0755,root,root) %{_datadir}/%{name}-common/scripts/network/cisco +%config(noreplace) %{_sysconfdir}/%{name}/agent +%dir %{_localstatedir}/log/%{name}/agent +%attr(0644,root,root) %{_datadir}/%{name}-agent/lib/*.jar +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/libvirtqemuhook +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/rolling-maintenance +%dir %{_datadir}/%{name}-agent/plugins +%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE +%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE + +%files common +%dir %attr(0755,root,root) %{_datadir}/%{name}-common/python-site/cloudutils +%dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms +%attr(0755,root,root) %{_datadir}/%{name}-common/scripts +%attr(0755,root,root) /usr/bin/cloudstack-sccs +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/agent.zip +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/cloud-scripts.tgz +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/patch-sysvms.sh +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloud_utils.py +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/__pycache__/* +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloudutils/* +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/%{name}-utils.jar +%{_defaultdocdir}/%{name}-common-%{version}/LICENSE +%{_defaultdocdir}/%{name}-common-%{version}/NOTICE + +%files ui +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/ui/config.json +%{_datadir}/%{name}-ui/* +%{_defaultdocdir}/%{name}-ui-%{version}/LICENSE +%{_defaultdocdir}/%{name}-ui-%{version}/NOTICE + +%files usage +%attr(0644,root,root) %{_unitdir}/%{name}-usage.service +%config(noreplace) %{_sysconfdir}/default/%{name}-usage +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage +%attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar +%attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar +%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage +%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/db.properties +%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/log4j-cloud.xml +%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE +%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE + +%files marvin +%attr(0644,root,root) %{_datadir}/%{name}-marvin/Marvin*.tar.gz +%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE +%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE + +%files integration-tests +%attr(0755,root,root) %{_datadir}/%{name}-integration-tests/* +%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE +%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE + +%if "%{_ossnoss}" == "noredist" +%files mysql-ha +%defattr(0644,cloud,cloud,0755) +%attr(0644,root,root) %{_datadir}/%{name}-mysql-ha/lib/* +%endif + +%files baremetal-agent +%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal + +%changelog +* Thu Dec 22 2022 Rohit Yadav 4.18.0 +- Add support for EL9 + +* Fri Oct 14 2022 Daan Hoogland 4.18.0 +- initialising sanity check pointer file + +* Tue Jun 29 2021 David Jumani 4.16.0 +- Adding SUSE 15 support + +* Thu Apr 30 2015 Rohit Yadav 4.6.0 +- Remove awsapi package + +* Wed Nov 19 2014 Hugo Trippaers 4.6.0 +- Create a specific spec for CentOS 7 + +* Fri Jul 4 2014 Hugo Trippaers 4.5.0 +- Add a package for the mysql ha module + +* Fri Oct 5 2012 Hugo Trippaers 4.1.0 +- new style spec file diff --git a/packaging/suse15/cloudstack-agent.te b/packaging/suse15/cloudstack-agent.te new file mode 120000 index 000000000000..30e123f6cba5 --- /dev/null +++ b/packaging/suse15/cloudstack-agent.te @@ -0,0 +1 @@ +../el8/cloudstack-agent.te \ No newline at end of file diff --git a/packaging/suse15/cloudstack-sccs b/packaging/suse15/cloudstack-sccs new file mode 120000 index 000000000000..b9e6ed9dc085 --- /dev/null +++ b/packaging/suse15/cloudstack-sccs @@ -0,0 +1 @@ +../el8/cloudstack-sccs \ No newline at end of file diff --git a/packaging/suse15/filelimit.conf b/packaging/suse15/filelimit.conf new file mode 120000 index 000000000000..c71688ea640a --- /dev/null +++ b/packaging/suse15/filelimit.conf @@ -0,0 +1 @@ +../el8/filelimit.conf \ No newline at end of file diff --git a/packaging/suse15/replace.properties b/packaging/suse15/replace.properties new file mode 100644 index 000000000000..b1900af83406 --- /dev/null +++ b/packaging/suse15/replace.properties @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DBUSER=cloud +DBPW=cloud +DBROOTPW= +MSLOG=vmops.log +APISERVERLOG=api.log +DBHOST=localhost +DBDRIVER=jdbc:mysql +COMPONENTS-SPEC=components-premium.xml +REMOTEHOST=localhost +AGENTCLASSPATH= +AGENTLOG=/var/log/cloudstack/agent/agent.log +AGENTLOGDIR=/var/log/cloudstack/agent/ +AGENTSYSCONFDIR=/etc/cloudstack/agent +APISERVERLOG=/var/log/cloudstack/management/apilog.log +BINDIR=/usr/bin +COMMONLIBDIR=/usr/share/cloudstack-common +CONFIGUREVARS= +DEPSCLASSPATH= +DOCDIR= +IPALOCATORLOG=/var/log/cloudstack/management/ipallocator.log +JAVADIR=/usr/share/java +LIBEXECDIR=/usr/libexec +LOCKDIR=/var/lock +MSCLASSPATH= +MSCONF=/etc/cloudstack/management +MSENVIRON=/usr/share/cloudstack-management +MSLOG=/var/log/cloudstack/management/management-server.log +MSLOGDIR=/var/log/cloudstack/management/ +MSMNTDIR=/var/cloudstack/mnt +MSUSER=cloud +PIDDIR=/var/run +PLUGINJAVADIR=/usr/share/cloudstack-management/plugin +PREMIUMJAVADIR=/usr/share/cloudstack-management/premium +PYTHONDIR=/usr/share/cloudstack-common/python-site/ +SERVERSYSCONFDIR=/etc/sysconfig +SETUPDATADIR=/usr/share/cloudstack-management/setup +SYSCONFDIR=/etc/sysconfig +SYSTEMCLASSPATH= +SYSTEMJARS= +USAGECLASSPATH= +USAGELOG=/var/log/cloudstack/usage/usage.log +USAGESYSCONFDIR=/etc/sysconfig +EXTENSIONSDEPLOYMENTMODE=production +GUESTNVRAMTEMPLATELEGACY=/usr/share/qemu/ovmf-x86_64-vars.bin +GUESTLOADERLEGACY=/usr/share/qemu/ovmf-x86_64-code.bin +GUESTNVRAMTEMPLATESECURE=/usr/share/qemu/ovmf-x86_64-ms-vars.bin +GUESTLOADERSECURE=/usr/share/qemu/ovmf-x86_64-ms-code.bin +GUESTNVRAMPATH=/var/lib/libvirt/qemu/nvram/ diff --git a/packaging/systemd/cloudstack-management.default b/packaging/systemd/cloudstack-management.default index 994a1ee86997..a41338beda68 100644 --- a/packaging/systemd/cloudstack-management.default +++ b/packaging/systemd/cloudstack-management.default @@ -17,7 +17,7 @@ JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED" -CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-mysql-ha/lib/*" +CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/cloudstack-mysql-ha/lib/*" BOOTSTRAP_CLASS=org.apache.cloudstack.ServerDaemon diff --git a/packaging/systemd/cloudstack-usage.default b/packaging/systemd/cloudstack-usage.default index 493f40c277a2..36b71ac3e0d6 100644 --- a/packaging/systemd/cloudstack-usage.default +++ b/packaging/systemd/cloudstack-usage.default @@ -17,7 +17,7 @@ JAVA_OPTS="-Xms256m -Xmx2048m --add-opens=java.base/java.lang=ALL-UNNAMED" -CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar" +CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage" JAVA_CLASS=com.cloud.usage.UsageServer diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index df9336026f4d..a7121f149e30 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -226,12 +226,12 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce } catch (AgentUnavailableException e) { logger.error("Unable to contact backend control plane to initiate backup for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { logger.error("Operation to initiate backup timed out for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } @@ -249,12 +249,12 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce } else { logger.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); if (answer.getNeedsCleanup()) { - logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state. Backup should be manually deleted to free up the space", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Error); backupDao.update(backupVO.getId(), backupVO); } else { backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); } return new Pair<>(false, null); } diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java index 5ff036fef12f..d018d488c64a 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java @@ -40,17 +40,17 @@ public final class RootCACustomTrustManager implements X509TrustManager { private boolean authStrictness = true; private boolean allowExpiredCertificate = true; private CrlDao crlDao; - private X509Certificate caCertificate; + private List caCertificates; private Map activeCertMap; - public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map activeCertMap, final X509Certificate caCertificate, final CrlDao crlDao) { + public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map activeCertMap, final List caCertificates, final CrlDao crlDao) { if (StringUtils.isNotEmpty(clientAddress)) { this.clientAddress = clientAddress.replace("/", "").split(":")[0]; } this.authStrictness = authStrictness; this.allowExpiredCertificate = allowExpiredCertificate; this.activeCertMap = activeCertMap; - this.caCertificate = caCertificate; + this.caCertificates = caCertificates; this.crlDao = crlDao; } @@ -151,6 +151,6 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr @Override public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[]{caCertificate}; + return caCertificates.toArray(new X509Certificate[0]); } } diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java index 25c45ed2a102..afb4f561160e 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -40,7 +40,6 @@ import java.security.spec.InvalidKeySpecException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Enumeration; import java.util.HashSet; import java.util.List; @@ -60,6 +59,7 @@ import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.ValidatedConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.KeyStoreUtils; @@ -92,6 +92,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con private static KeyPair caKeyPair = null; private static X509Certificate caCertificate = null; + private static List caCertificates = null; private static KeyStore managementKeyStore = null; @Inject @@ -103,20 +104,25 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con /////////////// Root CA Settings /////////////////// //////////////////////////////////////////////////// - private static ConfigKey rootCAPrivateKey = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.private.key", - null, - "The ROOT CA private key.", true); - - private static ConfigKey rootCAPublicKey = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.public.key", - null, - "The ROOT CA public key.", true); - - private static ConfigKey rootCACertificate = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.ca.certificate", - null, - "The ROOT CA certificate.", true); + private static ConfigKey rootCAPrivateKey = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.private.key", null, + "The ROOT CA private key in PEM format. " + + "When set along with the public key and certificate, CloudStack uses this custom CA instead of auto-generating one. " + + "All three ca.plugin.root.* keys must be set together. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validatePrivateKeyPem); + + private static ConfigKey rootCAPublicKey = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.public.key", null, + "The ROOT CA public key in PEM format (X.509/SPKI: must start with '-----BEGIN PUBLIC KEY-----'). " + + "Required when providing a custom CA. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validatePublicKeyPem); + + private static ConfigKey rootCACertificate = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.ca.certificate", null, + "The CA certificate(s) in PEM format (must start with '-----BEGIN CERTIFICATE-----'). " + + "For intermediate CAs, concatenate the signing cert first, followed by intermediate(s) and root. " + + "Required when providing a custom CA. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validateCACertificatePem); private static ConfigKey rootCAIssuerDN = new ConfigKey<>("Advanced", String.class, "ca.plugin.root.issuer.dn", @@ -151,7 +157,7 @@ private Certificate generateCertificate(final List domainNames, final Li caCertificate, caKeyPair, keyPair.getPublic(), subject, CAManager.CertSignatureAlgorithm.value(), validityDays, domainNames, ipAddresses); - return new Certificate(clientCertificate, keyPair.getPrivate(), Collections.singletonList(caCertificate)); + return new Certificate(clientCertificate, keyPair.getPrivate(), caCertificates); } private Certificate generateCertificateUsingCsr(final String csr, final List names, final List ips, final int validityDays) throws NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, CertificateException, SignatureException, IOException, OperatorCreationException { @@ -205,7 +211,7 @@ private Certificate generateCertificateUsingCsr(final String csr, final List getCaCertificate() { - return Collections.singletonList(caCertificate); + return caCertificates; } @Override @@ -254,8 +260,8 @@ public boolean revokeCertificate(final BigInteger certSerial, final String certC private KeyStore getCaKeyStore() throws CertificateException, NoSuchAlgorithmException, IOException, KeyStoreException { final KeyStore ks = KeyStore.getInstance("JKS"); ks.load(null, null); - if (caKeyPair != null && caCertificate != null) { - ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), new X509Certificate[]{caCertificate}); + if (caKeyPair != null && CollectionUtils.isNotEmpty(caCertificates)) { + ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), caCertificates.toArray(new X509Certificate[0])); } else { return null; } @@ -274,7 +280,7 @@ public SSLEngine createSSLEngine(final SSLContext sslContext, final String remot final boolean authStrictness = rootCAAuthStrictness.value(); final boolean allowExpiredCertificate = rootCAAllowExpiredCert.value(); - TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificate, crlDao)}; + TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificates, crlDao)}; sslContext.init(kmf.getKeyManagers(), tms, new SecureRandom()); final SSLEngine sslEngine = sslContext.createSSLEngine(); @@ -316,33 +322,39 @@ private boolean saveNewRootCAKeypair() { if (!configDao.update(rootCAPrivateKey.key(), rootCAPrivateKey.category(), CertUtils.privateKeyToPem(keyPair.getPrivate()))) { logger.error("Failed to save RootCA private key"); } + caKeyPair = keyPair; } catch (final NoSuchProviderException | NoSuchAlgorithmException | IOException e) { logger.error("Failed to generate/save RootCA private/public keys due to exception:", e); } - return loadRootCAKeyPair(); + return caKeyPair != null && caKeyPair.getPrivate() != null && caKeyPair.getPublic() != null; } - private boolean saveNewRootCACertificate() { + boolean saveNewRootCACertificate() { if (caKeyPair == null) { throw new CloudRuntimeException("Cannot issue self-signed root CA certificate as CA keypair is not initialized"); } try { logger.debug("Generating root CA certificate"); - final X509Certificate rootCaCertificate = CertUtils.generateV3Certificate( + final X509Certificate generatedCACert = CertUtils.generateV3Certificate( null, caKeyPair, caKeyPair.getPublic(), rootCAIssuerDN.value(), CAManager.CertSignatureAlgorithm.value(), getCaValidityDays(), null, null); - if (!configDao.update(rootCACertificate.key(), rootCACertificate.category(), CertUtils.x509CertificateToPem(rootCaCertificate))) { + if (!configDao.update(rootCACertificate.key(), rootCACertificate.category(), CertUtils.x509CertificateToPem(generatedCACert))) { logger.error("Failed to update RootCA public/x509 certificate"); } + caCertificates = new ArrayList<>(java.util.Collections.singletonList(generatedCACert)); + caCertificate = generatedCACert; } catch (final CertificateException | NoSuchAlgorithmException | NoSuchProviderException | SignatureException | InvalidKeyException | OperatorCreationException | IOException e) { logger.error("Failed to generate RootCA certificate from private/public keys due to exception:", e); return false; } - return loadRootCACertificate(); + return caCertificate != null; } private boolean loadRootCAKeyPair() { + if (caKeyPair != null) { + return true; + } if (StringUtils.isAnyEmpty(rootCAPublicKey.value(), rootCAPrivateKey.value())) { return false; } @@ -355,14 +367,35 @@ private boolean loadRootCAKeyPair() { return caKeyPair.getPrivate() != null && caKeyPair.getPublic() != null; } - private boolean loadRootCACertificate() { + boolean loadRootCACertificate() { + if (caCertificate != null && CollectionUtils.isNotEmpty(caCertificates)) { + return true; + } + caCertificate = null; + caCertificates = null; if (StringUtils.isEmpty(rootCACertificate.value())) { return false; } try { - caCertificate = CertUtils.pemToX509Certificate(rootCACertificate.value()); - caCertificate.verify(caKeyPair.getPublic()); - } catch (final IOException | CertificateException | NoSuchAlgorithmException | InvalidKeyException | SignatureException | NoSuchProviderException e) { + final List loadedCerts = CertUtils.pemToX509Certificates(rootCACertificate.value()); + if (CollectionUtils.isEmpty(loadedCerts)) { + logger.error("No certificates found in ca.plugin.root.ca.certificate"); + return false; + } + final X509Certificate loadedCACert = loadedCerts.get(0); + + // Verify key ownership without enforcing self-signature + if (!loadedCACert.getPublicKey().equals(caKeyPair.getPublic())) { + logger.error("The public key in the CA certificate does not match the configured CA public key"); + return false; + } + + if (loadedCerts.size() > 1) { + logger.info("Loaded CA certificate chain with {} certificate(s)", loadedCerts.size()); + } + caCertificates = loadedCerts; + caCertificate = loadedCACert; + } catch (final IOException | CertificateException e) { logger.error("Failed to load saved RootCA certificate due to exception:", e); return false; } @@ -389,9 +422,15 @@ private boolean loadManagementKeyStore() { try { managementKeyStore = KeyStore.getInstance("JKS"); managementKeyStore.load(null, null); - managementKeyStore.setCertificateEntry(caAlias, caCertificate); + int caIndex = 0; + for (final X509Certificate cert : caCertificates) { + managementKeyStore.setCertificateEntry(caAlias + "-" + caIndex++, cert); + } + final List fullChain = new ArrayList<>(); + fullChain.add(serverCertificate.getClientCertificate()); + fullChain.addAll(caCertificates); managementKeyStore.setKeyEntry(managementAlias, serverCertificate.getPrivateKey(), getKeyStorePassphrase(), - new X509Certificate[]{serverCertificate.getClientCertificate(), caCertificate}); + fullChain.toArray(new X509Certificate[0])); } catch (final CertificateException | NoSuchAlgorithmException | KeyStoreException | IOException e) { logger.error("Failed to load root CA management-server keystore due to exception: ", e); return false; @@ -421,14 +460,63 @@ protected void addConfiguredManagementIp(List ipList) { } + private static void validatePrivateKeyPem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + CertUtils.pemToPrivateKey(value); + } catch (InvalidKeySpecException | IOException e) { + throw new IllegalArgumentException( + "ca.plugin.root.private.key is not a valid PEM private key: " + e.getMessage()); + } + } + + private static void validatePublicKeyPem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + CertUtils.pemToPublicKey(value); + } catch (InvalidKeySpecException | IOException e) { + throw new IllegalArgumentException( + "ca.plugin.root.public.key is not a valid PEM public key: " + e.getMessage()); + } + } + + static void validateCACertificatePem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + final List certs = CertUtils.pemToX509Certificates(value); + if (CollectionUtils.isEmpty(certs)) { + throw new IllegalArgumentException( + "ca.plugin.root.ca.certificate contains no certificates"); + } + } catch (IOException | CertificateException e) { + throw new IllegalArgumentException( + "ca.plugin.root.ca.certificate is not a valid PEM certificate: " + e.getMessage()); + } + } + private boolean setupCA() { - if (!loadRootCAKeyPair() && !saveNewRootCAKeypair()) { - logger.error("Failed to save and load root CA keypair"); - return false; + if (!loadRootCAKeyPair()) { + if (hasUserProvidedCAKeys()) { + logger.error("Failed to load user-provided CA keys from configuration. " + + "Check that ca.plugin.root.private.key, ca.plugin.root.public.key, and " + + "ca.plugin.root.ca.certificate are all set and in the correct PEM format. " + + "Overwriting with auto-generated keys."); + } + if (!saveNewRootCAKeypair()) { + logger.error("Failed to save and load root CA keypair"); + return false; + } } - if (!loadRootCACertificate() && !saveNewRootCACertificate()) { - logger.error("Failed to save and load root CA certificate"); - return false; + if (!loadRootCACertificate()) { + if (hasUserProvidedCAKeys()) { + logger.error("Failed to load user-provided CA certificate. " + + "Check that ca.plugin.root.ca.certificate is set and in PEM format. " + + "Overwriting with auto-generated certificate."); + } + if (!saveNewRootCACertificate()) { + logger.error("Failed to save and load root CA certificate"); + return false; + } } if (!loadManagementKeyStore()) { logger.error("Failed to check and configure management server keystore"); @@ -437,10 +525,16 @@ private boolean setupCA() { return true; } + private boolean hasUserProvidedCAKeys() { + return StringUtils.isNotEmpty(rootCAPublicKey.value()) + || StringUtils.isNotEmpty(rootCAPrivateKey.value()) + || StringUtils.isNotEmpty(rootCACertificate.value()); + } + @Override public boolean start() { managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value(); - return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore(); + return loadRootCAKeyPair() && loadRootCACertificate() && loadManagementKeyStore(); } @Override diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java index d4ded3023321..714e18c3449f 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java @@ -23,9 +23,11 @@ import java.security.KeyPair; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.List; import org.apache.cloudstack.utils.security.CertUtils; import org.junit.Assert; @@ -63,14 +65,14 @@ public void setUp() throws Exception { @Test public void testAuthNotStrictWithInvalidCert() throws Exception { - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(null, null); } @Test public void testAuthNotStrictWithRevokedCert() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(new CrlVO()); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), caCertificate); @@ -79,7 +81,7 @@ public void testAuthNotStrictWithRevokedCert() throws Exception { @Test public void testAuthNotStrictWithInvalidCertOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), caCertificate); @@ -88,14 +90,14 @@ public void testAuthNotStrictWithInvalidCertOwnership() throws Exception { @Test(expected = CertificateException.class) public void testAuthNotStrictWithDenyExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, false, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, false, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); } @Test public void testAuthNotStrictWithAllowExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), expiredClientCertificate); @@ -103,35 +105,50 @@ public void testAuthNotStrictWithAllowExpiredCertAndOwnership() throws Exception @Test(expected = CertificateException.class) public void testAuthStrictWithInvalidCert() throws Exception { - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(null, null); } @Test(expected = CertificateException.class) public void testAuthStrictWithRevokedCert() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(new CrlVO()); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); } @Test(expected = CertificateException.class) public void testAuthStrictWithInvalidCertOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); } @Test(expected = CertificateException.class) public void testAuthStrictWithDenyExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, false, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, false, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); } + @Test + public void testGetAcceptedIssuersWithChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final List chain = Arrays.asList(caCertificate, rootCert); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager( + clientIp, false, true, certMap, chain, crlDao); + + final X509Certificate[] issuers = trustManager.getAcceptedIssuers(); + Assert.assertEquals(2, issuers.length); + Assert.assertEquals(caCertificate, issuers[0]); + Assert.assertEquals(rootCert, issuers[1]); + } + @Test public void testAuthStrictWithAllowExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); Assert.assertTrue(trustManager.getAcceptedIssuers() != null); Assert.assertTrue(trustManager.getAcceptedIssuers().length == 1); Assert.assertEquals(trustManager.getAcceptedIssuers()[0], caCertificate); diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java index 8311f4d45abc..21f00c66a1df 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -38,6 +39,7 @@ import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.SSLUtils; import org.bouncycastle.asn1.x509.GeneralName; @@ -49,7 +51,6 @@ import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.test.util.ReflectionTestUtils; @RunWith(MockitoJUnitRunner.class) @@ -75,7 +76,7 @@ public void setUp() throws Exception { addField(provider, "caKeyPair", caKeyPair); addField(provider, "caCertificate", caCertificate); - addField(provider, "caKeyPair", caKeyPair); + addField(provider, "caCertificates", Collections.singletonList(caCertificate)); } @After @@ -129,6 +130,46 @@ public void testIssueCertificateWithCsr() throws NoSuchProviderException, Certif certificate.getClientCertificate().verify(caCertificate.getPublicKey()); } + @Test + public void testGetCaCertificateWithChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final KeyPair intermediateKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate intermediateCert = CertUtils.generateV3Certificate(rootCert, rootKeyPair, + intermediateKeyPair.getPublic(), "CN=intermediate", "SHA256withRSA", 365, null, null); + + final List chain = Arrays.asList(intermediateCert, rootCert); + addField(provider, "caKeyPair", intermediateKeyPair); + addField(provider, "caCertificate", intermediateCert); + addField(provider, "caCertificates", chain); + + Assert.assertEquals(2, provider.getCaCertificate().size()); + Assert.assertEquals(intermediateCert, provider.getCaCertificate().get(0)); + Assert.assertEquals(rootCert, provider.getCaCertificate().get(1)); + } + + @Test + public void testIssueCertificateWithoutCsrAndChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final KeyPair intermediateKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate intermediateCert = CertUtils.generateV3Certificate(rootCert, rootKeyPair, + intermediateKeyPair.getPublic(), "CN=intermediate", "SHA256withRSA", 365, null, null); + + addField(provider, "caKeyPair", intermediateKeyPair); + addField(provider, "caCertificate", intermediateCert); + addField(provider, "caCertificates", Arrays.asList(intermediateCert, rootCert)); + + final Certificate certificate = provider.issueCertificate(Arrays.asList("domain1.com"), null, 1); + Assert.assertNotNull(certificate); + Assert.assertEquals(2, certificate.getCaCertificates().size()); + Assert.assertEquals(intermediateCert, certificate.getCaCertificates().get(0)); + Assert.assertEquals(rootCert, certificate.getCaCertificates().get(1)); + certificate.getClientCertificate().verify(intermediateKeyPair.getPublic()); + } + @Test public void testRevokeCertificate() throws Exception { Assert.assertTrue(provider.revokeCertificate(CertUtils.generateRandomBigInt(), "anyString")); @@ -177,8 +218,8 @@ public void testIsManagementCertificateNoAltNames() { } @Test - public void testIsManagementCertificateNoMatch() { - ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", "cloudstack"); + public void testIsManagementCertificateNoMatch() throws Exception { + addField(provider, "managementCertificateCustomSAN", "cloudstack"); try { X509Certificate certificate = Mockito.mock(X509Certificate.class); List> altNames = new ArrayList<>(); @@ -193,9 +234,9 @@ public void testIsManagementCertificateNoMatch() { } @Test - public void testIsManagementCertificateMatch() { + public void testIsManagementCertificateMatch() throws Exception { String customSAN = "cloudstack"; - ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", customSAN); + addField(provider, "managementCertificateCustomSAN", customSAN); try { X509Certificate certificate = Mockito.mock(X509Certificate.class); List> altNames = new ArrayList<>(); @@ -208,4 +249,58 @@ public void testIsManagementCertificateMatch() { Assert.fail(String.format("Exception occurred: %s", e.getMessage())); } } + + @Test + public void testLoadRootCACertificateWithMismatchedCert() throws Exception { + KeyPair otherKeyPair = CertUtils.generateRandomKeyPair(1024); + X509Certificate mismatchedCert = CertUtils.generateV3Certificate(null, otherKeyPair, otherKeyPair.getPublic(), "CN=other", "SHA256withRSA", 365, null, null); + String mismatchedPem = CertUtils.x509CertificateToPem(mismatchedCert); + + ConfigKey mockCertKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockCertKey.value()).thenReturn(mismatchedPem); + addField(provider, "rootCACertificate", mockCertKey); + + addField(provider, "caCertificate", null); + addField(provider, "caCertificates", null); + + Boolean result = provider.loadRootCACertificate(); + Assert.assertFalse(result); + Assert.assertNull(provider.getCaCertificate()); + } + + @Test + public void testSaveNewRootCACertificateWithStaleCache() throws Exception { + ConfigurationDao configDao = Mockito.mock(ConfigurationDao.class); + addField(provider, "configDao", configDao); + + ConfigKey mockCertKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockCertKey.key()).thenReturn("ca.plugin.root.ca.certificate"); + Mockito.when(mockCertKey.category()).thenReturn("Hidden"); + addField(provider, "rootCACertificate", mockCertKey); + + ConfigKey mockIssuerKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockIssuerKey.value()).thenReturn("CN=ca.cloudstack.apache.org"); + addField(provider, "rootCAIssuerDN", mockIssuerKey); + + addField(provider, "caCertificate", null); + addField(provider, "caCertificates", null); + + Mockito.when(configDao.update(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(true); + + Boolean result = provider.saveNewRootCACertificate(); + Assert.assertTrue(result); + Assert.assertNotNull(provider.getCaCertificate()); + Assert.assertEquals(1, provider.getCaCertificate().size()); + } + + @Test + public void testValidateCACertificatePem() throws Exception { + String truncatedPem = "-----BEGIN CERTIFICATE-----\nMIICxTCCAa0CAQAw\n"; + try { + RootCAProvider.validateCACertificatePem(truncatedPem); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("is not a valid PEM certificate")); + } + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 0cec0df66182..e5c89c5de1c5 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -17,12 +17,9 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; -import com.cloud.user.Account; - import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -30,36 +27,41 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.api.response.QuotaStatementItemResponse; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaBalance", responseObject = QuotaStatementItemResponse.class, description = "Create a quota balance statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, +@APICommand(name = "quotaBalance", responseObject = QuotaBalanceResponse.class, description = "Create Quota balance statements for the provided Accounts or Projects.", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaBalanceCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Name of the Account for which balance statements will be generated. " + + "Deprecated, please use 'accountid' instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "ID of the domain which the Account identified by 'account' belongs to." + + "Deprecated, please use 'accountid' instead.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End of the period of the Quota balance." + - ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "Date of the last Quota balance to be returned. Must be informed together with the " + + "startdate parameter, and can not be before startdate. " + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start of the period of the Quota balance. " + + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Date of the first Quota balance to be returned. Must be before today. " + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which balance statements will be generated.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which balance statements will be generated. Can not be specified with accountid.") + private Long projectId; + @Inject QuotaResponseBuilder _responseBuilder; @@ -88,48 +90,32 @@ public void setDomainId(Long domainId) { } public Date getEndDate() { - if (endDate == null){ - return null; - } - else{ - return _responseBuilder.startOfNextDay(new Date(endDate.getTime())); - } + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; - } - Account account = _accountService.getActiveAccountByName(accountName, domainId); - if (account != null) { - return account.getAccountId(); + if (ObjectUtils.allNull(accountId, accountName, domainId, projectId)) { + return -1; } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaBalance(this); - - QuotaBalanceResponse response; - if (endDate == null) { - response = _responseBuilder.createQuotaLastBalanceResponse(quotaUsage, getStartDate()); - } else { - response = _responseBuilder.createQuotaBalanceResponse(quotaUsage, getStartDate(), new Date(endDate.getTime())); - } + QuotaBalanceResponse response = _responseBuilder.createQuotaBalanceResponse(this); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaResourceStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaResourceStatementCmd.java new file mode 100644 index 000000000000..00dcb8fa5752 --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaResourceStatementCmd.java @@ -0,0 +1,118 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.command; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.QuotaResponseBuilder; +import org.apache.cloudstack.api.response.QuotaResourceStatementResponse; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; + +import javax.inject.Inject; + +import java.util.Date; + +@APICommand(name = "quotaResourceStatement", responseObject = QuotaResourceStatementResponse.class, since = "4.23.0.0", + description = "Generates a detailed Quota statement for a specific resource.", requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false) +public class QuotaResourceStatementCmd extends BaseCmd { + + @Parameter(name = ApiConstants.ID, type = CommandType.STRING, required = true, description = "ID of the resource.") + private String id; + + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "Generate the statement for this Account. A resource may have belonged to different owners at distinct points in time, " + + "so this parameter can be used to only consider the period for which it belonged to a specific Account.") + private Long accountId; + + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "Generate the statement for this Project. A resource may have belonged to different owners at distinct points in time, " + + "so this parameter can be used to only consider the period for which it belonged to a specific Project.") + private Long projectId; + + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "ID of the Domain for which the Quota statement will be generated.") + private Long domainId; + + @Parameter(name = ApiConstants.USAGE_TYPE, type = CommandType.INTEGER, required = true, description = "Usage type of the Quota usage.") + private Integer usageType; + + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, required = true, description = "Start date for the query. " + + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) + private Date startDate; + + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End date for the query. " + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) + private Date endDate; + + @Parameter(name = ApiConstants.IS_RECURSIVE, type = CommandType.BOOLEAN, description = "Whether to include usage records from children of the filtered domain. " + + " Defaults to false.") + private Boolean recursive; + + @Inject + QuotaResponseBuilder responseBuilder; + + @Override + public void execute() { + QuotaResourceStatementResponse response = responseBuilder.createQuotaResourceStatement(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + if (ObjectUtils.allNull(accountId, projectId)) { + return -1; + } + return _accountService.finalizeAccountId(accountId, null, null, projectId); + } + + public String getId() { + return id; + } + + public Integer getUsageType() { + return usageType; + } + + public Date getStartDate() { + return startDate; + } + + public Date getEndDate() { + return endDate; + } + + public Long getAccountId() { + return accountId; + } + + public Long getDomainId() { + return domainId; + } + + public boolean isRecursive() { + return BooleanUtils.isTrue(recursive); + } + +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index bfe26a9f4250..f3a27ea58718 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -20,7 +20,6 @@ import javax.inject.Inject; -import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; @@ -32,18 +31,17 @@ import org.apache.cloudstack.api.response.QuotaStatementItemResponse; import org.apache.cloudstack.api.response.QuotaStatementResponse; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; @APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a Quota statement for the provided Account, Project, or Domain.", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { - @ACL @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Name of the Account for which the Quota statement will be generated. Deprecated, please use accountid instead.") private String accountName; - @ACL @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "ID of the Domain for which the Quota statement will be generated. May be used individually or with account.") private Long domainId; @@ -60,16 +58,18 @@ public class QuotaStatementCmd extends BaseCmd { description = "Consider only Quota usage records for the specified usage type in the statement.") private Integer usageType; - @ACL @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which the Quota statement will be generated. Can not be specified with projectid.") private Long accountId; - @ACL @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which the Quota statement will be generated. Can not be specified with accountid.", since = "4.23.0") private Long projectId; + @Parameter(name = ApiConstants.IS_RECURSIVE, type = CommandType.BOOLEAN, description = "Whether to include usage records from children of the filtered domain. " + + " Defaults to false.") + private Boolean recursive; + @Parameter(name = ApiConstants.SHOW_RESOURCES, type = CommandType.BOOLEAN, description = "List the resources of each Quota type in the period.", since = "4.23.0") private boolean showResources; @@ -152,4 +152,9 @@ public void execute() { response.setResponseName(getCommandName()); setResponseObject(response); } + + public boolean isRecursive() { + return BooleanUtils.isTrue(recursive); + } + } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java index 625f4829c673..714066b048ad 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java @@ -17,137 +17,65 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; import java.util.Date; import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import com.cloud.serializer.Param; public class QuotaBalanceResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; - - @SerializedName("startquota") - @Param(description = "Quota started with") - private BigDecimal startQuota; - - @SerializedName("endquota") - @Param(description = "Quota by end of this period") - private BigDecimal endQuota; - - @SerializedName("credits") - @Param(description = "List of credits made during this period") - private List credits = null; - - @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; - - @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; - - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Balance's currency.") private String currency; - public QuotaBalanceResponse() { - super(); - credits = new ArrayList(); - } - - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } + @SerializedName(ApiConstants.DATE) + @Param(description = "Balance's date.") + private Date date; - public void setAccountName(String accountName) { - this.accountName = accountName; - } + @SerializedName(ApiConstants.BALANCE) + @Param(description = "Balance's value.") + private BigDecimal balance; - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } + @SerializedName(ApiConstants.BALANCES) + @Param(description = "List of balances in the period.") + private List balances; - public BigDecimal getStartQuota() { - return startQuota; - } - - public void setStartQuota(BigDecimal startQuota) { - this.startQuota = startQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public BigDecimal getEndQuota() { - return endQuota; - } - - public void setEndQuota(BigDecimal endQuota) { - this.endQuota = endQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public List getCredits() { - return credits; - } - - public void setCredits(List credits) { - this.credits = credits; + public QuotaBalanceResponse() { + super("balance"); } - public void addCredits(QuotaBalanceVO credit) { - QuotaCreditsResponse cr = new QuotaCreditsResponse(); - cr.setCredit(credit.getCreditBalance()); - cr.setCreditedOn(credit.getUpdatedOn()); - credits.add(0, cr); + public QuotaBalanceResponse(Date date, BigDecimal balance) { + super("balance"); + this.date = date; + this.balance = balance; } - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + public void setCurrency(String currency) { + this.currency = currency; } - public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + public void setBalances(List balances) { + this.balances = balances; } - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + public String getCurrency() { + return currency; } - public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + public List getBalances() { + return balances; } - public String getCurrency() { - return currency; + public Date getDate() { + return date; } - public void setCurrency(String currency) { - this.currency = currency; + public BigDecimal getBalance() { + return balance; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResourceStatementItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResourceStatementItemResponse.java new file mode 100644 index 000000000000..9f0a28389a9d --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResourceStatementItemResponse.java @@ -0,0 +1,82 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import java.math.BigDecimal; +import java.util.Date; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class QuotaResourceStatementItemResponse extends BaseResponse { + + @SerializedName(ApiConstants.TARIFF_ID) + @Param(description = "ID of the tariff.") + private String tariffId; + + @SerializedName(ApiConstants.TARIFF_NAME) + @Param(description = "Name of the tariff.") + private String tariffName; + + @SerializedName(ApiConstants.QUOTA_CONSUMED) + @Param(description = "Amount of quota used.") + private BigDecimal quotaUsed; + + @SerializedName(ApiConstants.START_DATE) + @Param(description = "Item's start date.") + private Date startDate; + + @SerializedName(ApiConstants.END_DATE) + @Param(description = "Item's end date.") + private Date endDate; + + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "UUID of the resource's owner.") + private String accountId; + + public QuotaResourceStatementItemResponse() { + super("quotaresourcestatementitem"); + } + + public void setTariffId(String tariffId) { + this.tariffId = tariffId; + } + + public void setTariffName(String tariffName) { + this.tariffName = tariffName; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public void setAccountId(String accountId) { + this.accountId = accountId; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResourceStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResourceStatementResponse.java new file mode 100644 index 000000000000..ce3ab1f177ba --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResourceStatementResponse.java @@ -0,0 +1,66 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import java.math.BigDecimal; +import java.util.List; + +public class QuotaResourceStatementResponse extends BaseResponse { + + @SerializedName(ApiConstants.USAGE_NAME) + @Param(description = "Name of the usage type.") + private String usageName; + + @SerializedName(ApiConstants.UNIT) + @Param(description = "Unit of the usage type.") + private String unit; + + @SerializedName(ApiConstants.ITEMS) + @Param(description = "List of Quota tariff usages.", responseObject = QuotaResourceStatementItemResponse.class) + private List items; + + @SerializedName(ApiConstants.TOTAL_QUOTA) + @Param(description = "Total amount of quota used.") + private BigDecimal totalQuotaUsed; + + public QuotaResourceStatementResponse() { + super("quotaresourcestatement"); + } + + public void setUsageName(String usageName) { + this.usageName = usageName; + } + + public void setUnit(String unit) { + this.unit = unit; + } + + public void setQuotaUsageDetails(List items) { + this.items = items; + } + + public void setTotalQuotaUsed(BigDecimal totalQuotaUsed) { + this.totalQuotaUsed = totalQuotaUsed; + } + +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index bde905c487b7..63bf043f4fa9 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -23,13 +23,13 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; +import org.apache.cloudstack.api.command.QuotaResourceStatementCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; @@ -50,14 +50,10 @@ public interface QuotaResponseBuilder { QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); - QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); + QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd); Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); - QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - - List getQuotaBalance(QuotaBalanceCmd cmd); - QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); List listQuotaEmailTemplates(QuotaEmailTemplateListCmd cmd); @@ -88,4 +84,6 @@ public interface QuotaResponseBuilder { Pair, Integer> createQuotaCreditsListResponse(QuotaCreditsListCmd cmd); QuotaValidateActivationRuleResponse validateActivationRule(QuotaValidateActivationRuleCmd cmd); + + QuotaResourceStatementResponse createQuotaResourceStatement(QuotaResourceStatementCmd cmd); } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 173c0723731b..b71057e64234 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -26,13 +26,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -69,11 +66,14 @@ import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; @@ -81,6 +81,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; +import org.apache.cloudstack.api.command.QuotaResourceStatementCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; @@ -98,6 +99,7 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.ResourceCounting; import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -109,16 +111,20 @@ import org.apache.cloudstack.quota.dao.QuotaEmailTemplatesDao; import org.apache.cloudstack.quota.dao.QuotaSummaryDao; import org.apache.cloudstack.quota.dao.QuotaTariffDao; +import org.apache.cloudstack.quota.dao.QuotaTariffUsageDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaSummaryVO; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.cloudstack.quota.vo.QuotaUsageResourceVO; +import org.apache.cloudstack.usage.UsageTypes; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections4.CollectionUtils; @@ -183,9 +189,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private VMTemplateDao vmTemplateDao; @Inject private VolumeDao volumeDao; + @Inject + private QuotaUsageJoinDao quotaUsageJoinDao; + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; + @Inject + private EntityManager entityMgr; - - private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class, ResourceCounting.class}; private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); @@ -328,100 +339,25 @@ public boolean isUserAllowedToSeeActivationRules(User user) { } @Override - public QuotaBalanceResponse createQuotaBalanceResponse(List quotaBalance, Date startDate, Date endDate) { - if (quotaBalance == null || quotaBalance.isEmpty()) { - throw new InvalidParameterValueException("The request period does not contain balance entries."); - } - Collections.sort(quotaBalance, new Comparator() { - @Override - public int compare(QuotaBalanceVO o1, QuotaBalanceVO o2) { - o1 = o1 == null ? new QuotaBalanceVO() : o1; - o2 = o2 == null ? new QuotaBalanceVO() : o2; - return o2.getUpdatedOn().compareTo(o1.getUpdatedOn()); // desc - } - }); - - boolean have_balance_entries = false; - //check that there is at least one balance entry - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (entry.isBalanceEntry()) { - have_balance_entries = true; - break; - } - } - //if last entry is a credit deposit then remove that as that is already - //accounted for in the starting balance after that entry, note the sort is desc - if (have_balance_entries) { - ListIterator li = quotaBalance.listIterator(quotaBalance.size()); - // Iterate in reverse. - while (li.hasPrevious()) { - QuotaBalanceVO entry = li.previous(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - li.remove(); - } else { - break; - } - } - } + public QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd) { + List quotaBalances = _quotaService.listQuotaBalancesForAccount(cmd.getEntityOwnerId(), cmd.getStartDate(), cmd.getEndDate()); - int quota_activity = quotaBalance.size(); - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - boolean consecutive = true; - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: All Credit Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - if (consecutive) { - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.addCredits(entry); - it.remove(); - } else { - consecutive = false; - } - } + List balances = quotaBalances.stream() + .map(balance -> new QuotaBalanceResponse(balance.getUpdatedOn(), balance.getCreditBalance())) + .collect(Collectors.toList()); - if (quota_activity > 0 && quotaBalance.size() > 0) { - // order is desc last item is the start item - QuotaBalanceVO startItem = quotaBalance.get(quotaBalance.size() - 1); - QuotaBalanceVO endItem = quotaBalance.get(0); - resp.setStartDate(startDate); - resp.setStartQuota(startItem.getCreditBalance()); - resp.setEndDate(endDate); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Start Entry=" + startItem); - logger.debug("createQuotaBalanceResponse: End Entry=" + endItem); - } - resp.setEndQuota(endItem.getCreditBalance().add(lastCredits)); - } else if (quota_activity > 0) { - // order is desc last item is the start item - resp.setStartDate(startDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndDate(endDate); - resp.setEndQuota(new BigDecimal(0).add(lastCredits)); - } else { - resp.setStartDate(startDate); - resp.setEndDate(endDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndQuota(new BigDecimal(0)); - } - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; + QuotaBalanceResponse response = new QuotaBalanceResponse(); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setBalances(balances); + + return response; } @Override public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd) { - Long accountId = getAccountIdForQuotaStatement(cmd); - Long domainId = getDomainIdForQuotaStatement(cmd, accountId); - List quotaUsages = _quotaService.getQuotaUsage(accountId, null, domainId, cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); + Long accountId = getAccountIdForQuotaStatement(cmd.getEntityOwnerId(), null); + Pair> baseDomainAndFilteredDomains = getDomainIdsForQuotaStatement(accountId, cmd.getDomainId(), cmd.isRecursive()); + List quotaUsages = _quotaService.getQuotaUsage(accountId, null, baseDomainAndFilteredDomains.second(), cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); logger.debug("Creating quota statement from [{}] usage records for parameters [{}].", quotaUsages.size(), ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "accountId", "projectId", "domainId", "startDate", "endDate", "usageType", "showResources")); @@ -444,52 +380,16 @@ public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd Account account = _accountDao.findByIdIncludingRemoved(accountId); statement.setAccountId(account.getUuid()); statement.setAccountName(account.getAccountName()); - domainId = account.getDomainId(); } - if (domainId != null) { - DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + Long baseDomainId = baseDomainAndFilteredDomains.first(); + if (baseDomainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(baseDomainId); statement.setDomainId(domain.getUuid()); } return statement; } - protected Long getAccountIdForQuotaStatement(QuotaStatementCmd cmd) { - if (Account.Type.NORMAL.equals(CallContext.current().getCallingAccount().getType())) { - logger.debug("Limiting the Quota statement for the calling Account, as they are a User Account."); - return CallContext.current().getCallingAccountId(); - } - - long accountId = cmd.getEntityOwnerId(); - if (accountId != -1) { - return accountId; - } - - if (cmd.getDomainId() == null) { - logger.debug("Limiting the Quota statement for the calling Account, as 'domainid' was not informed."); - return CallContext.current().getCallingAccountId(); - } - - logger.debug("Allowing admin/domain admin to generate the Quota statement for the provided Domain."); - return null; - } - - protected Long getDomainIdForQuotaStatement(QuotaStatementCmd cmd, Long accountId) { - if (accountId != null) { - logger.debug("Quota statement is already limited to Account [{}].", accountId); - Account account = _accountDao.findByIdIncludingRemoved(accountId); - return account.getDomainId(); - } - - Long domainId = cmd.getDomainId(); - if (domainId != null) { - return domainId; - } - - logger.debug("Limiting the Quota statement for the calling Account's Domain."); - return CallContext.current().getCallingAccount().getDomainId(); - } - protected void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(List quotaUsages, Integer usageType) { if (usageType != null) { logger.debug("As the usage type [{}] was informed as parameter of the API quotaStatement, we will not create dummy records.", usageType); @@ -783,7 +683,7 @@ public QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Doubl throw new InvalidParameterValueException("Account does not exist with account id " + accountId); } final boolean lockAccountEnforcement = "true".equalsIgnoreCase(QuotaConfig.QuotaEnableEnforcement.value()); - final BigDecimal currentAccountBalance = _quotaBalanceDao.lastQuotaBalance(accountId, domainId, startOfNextDay(new Date(depositedOn.getTime()))); + final BigDecimal currentAccountBalance = _quotaBalanceDao.getLastQuotaBalance(accountId, domainId); logger.debug("Depositing [{}] credits on adjusted date [{}]; current balance is [{}].", amount, DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), depositedOn), currentAccountBalance); // update quota account with the balance @@ -847,34 +747,6 @@ public boolean updateQuotaEmailTemplate(QuotaEmailTemplateUpdateCmd cmd) { return false; } - @Override - public QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate) { - if (quotaBalance == null) { - throw new InvalidParameterValueException("There are no balance entries on or before the requested date."); - } - if (startDate == null) { - startDate = new Date(); - } - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - for (QuotaBalanceVO entry : quotaBalance) { - logger.debug("createQuotaLastBalanceResponse Date={} balance={} credit={}", - DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), entry.getUpdatedOn()), - entry.getCreditBalance(), entry.getCreditsId()); - - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.setStartQuota(lastCredits); - resp.setStartDate(startDate); - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; - } - - @Override - public List getQuotaBalance(QuotaBalanceCmd cmd) { - return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); - } @Override public Date startOfNextDay(Date date) { LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); @@ -1243,7 +1115,7 @@ public QuotaValidateActivationRuleResponse validateActivationRule(QuotaValidateA addAllPresetVariables(PresetVariables.class, quotaType, usageTypeVariablesAndDescriptions, null); List usageTypeVariables = usageTypeVariablesAndDescriptions.stream().map(Pair::first).collect(Collectors.toList()); - try (JsInterpreter jsInterpreter = new JsInterpreter(QuotaConfig.QuotaActivationRuleTimeout.value())) { + try (JsInterpreter jsInterpreter = new JsInterpreter(QuotaConfig.QuotaActivationRuleTimeout.value(), QuotaConfig.QuotaActivationRuleTimeout.key())) { Map newVariables = injectUsageTypeVariables(jsInterpreter, usageTypeVariables); String scriptToExecute = jsInterpreterHelper.replaceScriptVariables(activationRule, newVariables); jsInterpreter.executeScript(String.format("new Function(\"%s\")", scriptToExecute.replaceAll("\n", ""))); @@ -1263,6 +1135,184 @@ public QuotaValidateActivationRuleResponse validateActivationRule(QuotaValidateA return createValidateActivationRuleResponse(activationRule, quotaName, false, message); } + @Override + public QuotaResourceStatementResponse createQuotaResourceStatement(QuotaResourceStatementCmd cmd) { + String resourceUuid = cmd.getId(); + Integer usageType = cmd.getUsageType(); + Date startDate = cmd.getStartDate(); + Date endDate = cmd.getEndDate(); + + if (startDate.after(endDate)) { + throw new InvalidParameterValueException(String.format("The start date [%s] must be before the end date [%s].", startDate, endDate)); + } + + InternalIdentity resource = retrieveResource(resourceUuid, usageType); + if (resource == null) { + logger.error("Could not find resource [{}] of type [{}]. Returning an empty list.", resourceUuid, usageType); + return createQuotaResourceStatementResponse(resourceUuid, usageType, new ArrayList<>(), new BigDecimal(0)); + } + + long id = resource.getId(); + + Long currentOwnerId = null; + if (resource instanceof ControlledEntity) { + currentOwnerId = ((ControlledEntity) resource).getAccountId(); + } + Long accountId = getAccountIdForQuotaStatement(cmd.getEntityOwnerId(), currentOwnerId); + Pair> baseDomainAndFilteredDomains = getDomainIdsForQuotaStatement(accountId, cmd.getDomainId(), cmd.isRecursive()); + + Long resourceId = null; + Long networkId = null; + Long offeringId = null; + + switch (usageType) { + case UsageTypes.NETWORK_OFFERING: + case UsageTypes.BACKUP: + offeringId = id; + break; + case UsageTypes.NETWORK_BYTES_RECEIVED: + case UsageTypes.NETWORK_BYTES_SENT: + networkId = id; + break; + default: + resourceId = id; + } + + logger.debug("Attempting to find quota usages with parameters usage type [{}], usage id [{}], network id [{}], offering id [{}], and between [{}] and [{}].", + usageType, resourceId, networkId, offeringId, startDate, endDate); + + List quotaUsageJoinList = quotaUsageJoinDao.findQuotaUsage(accountId, baseDomainAndFilteredDomains.second(), usageType, resourceId, networkId, offeringId, startDate, endDate, null); + + logger.debug("Found [{}] quota usages using as parameter usage type [{}], usage id [{}], network id [{}], offering id [{}], and between [{}] and [{}].", + quotaUsageJoinList.size(), usageType, resourceId, networkId, offeringId, startDate, endDate); + + List quotaResourceStatementItemResponseList = new ArrayList<>(); + BigDecimal totalQuotaUsed = new BigDecimal(0); + List quotaTariffs = _quotaTariffDao.listQuotaTariffs(null, null, usageType, null, null, true, null, null).first(); + + for (QuotaUsageJoinVO quotaUsageJoin : quotaUsageJoinList) { + Account account = _accountMgr.getAccount(quotaUsageJoin.getAccountId()); + String accountUuid = account.getUuid(); + + List quotaTariffUsageList = quotaTariffUsageDao.listQuotaTariffUsages(quotaUsageJoin.getId()); + logger.debug("Found [{}] quota tariff usages associated to the quota usage [{}] of resource [{}] and type [{}] between [{}] and [{}].", + quotaTariffUsageList.size(), quotaUsageJoin, resourceUuid, usageType, startDate, endDate); + for (QuotaTariffUsageVO quotaTariffUsage: quotaTariffUsageList) { + quotaResourceStatementItemResponseList.add(createQuotaResourceStatementItemResponse(quotaTariffUsage, quotaTariffs, quotaUsageJoin.getStartDate(), + quotaUsageJoin.getEndDate(), accountUuid)); + totalQuotaUsed = totalQuotaUsed.add(quotaTariffUsage.getQuotaUsed()); + } + } + logger.debug("The total quota used of type [{}] between [{}] and [{}] for the resource [{}] was [{}].", usageType, startDate, endDate, resourceUuid, totalQuotaUsed); + + return createQuotaResourceStatementResponse(resourceUuid, usageType, quotaResourceStatementItemResponseList, totalQuotaUsed); + } + + protected Long getAccountIdForQuotaStatement(long providedAccountId, Long fallbackAccountId) { + Account caller = CallContext.current().getCallingAccount(); + + if (providedAccountId != -1L) { + Account account = _accountDao.findByIdIncludingRemoved(providedAccountId); + _accountMgr.checkAccess(caller, null, false, account); + logger.debug("Limiting the Quota resource statement for the provided Account [{}].", providedAccountId); + return providedAccountId; + } + + Account.Type callerType = caller.getType(); + if (Account.Type.ADMIN.equals(callerType) || Account.Type.DOMAIN_ADMIN.equals(callerType)) { + logger.debug("Not limiting the Quota resource statement for a specific Account, as no specific Account was provided and the caller is either an admin or domain admin."); + return null; + } + + if (fallbackAccountId != null) { + Account fallbackAccount = _accountDao.findByIdIncludingRemoved(fallbackAccountId); + _accountMgr.checkAccess(caller, null, false, fallbackAccount); + logger.debug("Limiting the Quota statement for the fallback Account [{}], as no specific Account was provided.", fallbackAccountId); + return fallbackAccountId; + } + + logger.debug("Limiting the Quota resource statement for the calling account, as no specific Account was provided and no fallback account was provided."); + return caller.getAccountId(); + } + + protected Pair> getDomainIdsForQuotaStatement(Long finalAccountId, Long providedDomainId, boolean isRecursive) { + if (finalAccountId != null) { + // Access to the provided account has already been validated + logger.debug("Not limiting the Quota statement for a specific Domain, as we are already limiting by Account."); + return new Pair<>(null, null); + } + + // User accounts will have already been limited to themselves + Account caller = CallContext.current().getCallingAccount(); + Long domainId = providedDomainId; + + if (domainId != null) { + Domain domain = domainDao.findByIdIncludingRemoved(domainId); + _accountMgr.checkAccess(caller, domain); + logger.debug("Limiting the Quota statement for the provided Domain [{}].", domainId); + } else { + domainId = caller.getDomainId(); + logger.debug("Limiting the Quota statement for the caller's Domain [{}], as no 'domainid' was provided.", domainId); + } + + if (isRecursive) { + logger.debug("Allowing the Quota statement for the Domain's children, as the 'isrecursive' parameter was provided."); + return new Pair<>(domainId, domainDao.getDomainAndChildrenIds(domainId)); + } + + return new Pair<>(domainId, List.of(domainId)); + } + + protected InternalIdentity retrieveResource(String resourceUuid, Integer usageType) { + Class clazz = QuotaTypes.getClazz(usageType); + if (clazz == null) { + throw new InvalidParameterValueException(String.format("Invalid usage type [%s] provided.", usageType)); + } + + logger.debug("Attempting to find a resource with ID [{}] and of type [{}].", resourceUuid, usageType); + Object object = entityMgr.findByUuidIncludingRemoved(clazz, resourceUuid); + if (object == null) { + return null; + } + return (InternalIdentity) object; + } + + protected QuotaResourceStatementItemResponse createQuotaResourceStatementItemResponse(QuotaTariffUsageVO quotaTariffUsage, List quotaTariffs, + Date startDate, Date endDate, String accountUuid) { + logger.trace("Creating quota resource statement item associated to quota tariff usage [{}].", quotaTariffUsage); + QuotaResourceStatementItemResponse quotaResourceStatementItemResponse = new QuotaResourceStatementItemResponse(); + + QuotaTariffVO quotaTariff = quotaTariffs.stream().filter(quotaTariffVO -> quotaTariffUsage.getTariffId().equals(quotaTariffVO.getId())).findAny().orElse(null); + + quotaResourceStatementItemResponse.setQuotaUsed(quotaTariffUsage.getQuotaUsed()); + quotaResourceStatementItemResponse.setStartDate(startDate); + quotaResourceStatementItemResponse.setEndDate(endDate); + quotaResourceStatementItemResponse.setAccountId(accountUuid); + if (quotaTariff != null) { + logger.trace("Quota usage details item will be associated to the quota tariff [{}].", quotaTariff); + quotaResourceStatementItemResponse.setTariffId(quotaTariff.getUuid()); + quotaResourceStatementItemResponse.setTariffName(quotaTariff.getName()); + } + + return quotaResourceStatementItemResponse; + } + + protected QuotaResourceStatementResponse createQuotaResourceStatementResponse(String resourceUuid, Integer usageType, + List quotaUsageDetailsItems, BigDecimal totalQuotaUsed) { + logger.trace("Creating quota usage details list response associated to the resource of UUID [{}], with an usage type of [{}], [{}] quota" + + " usage details items, and a total quota used of [{}].", resourceUuid, usageType, quotaUsageDetailsItems.size(), totalQuotaUsed); + QuotaResourceStatementResponse quotaResourceStatementResponse = new QuotaResourceStatementResponse(); + + QuotaTypes quotaType = QuotaTypes.getQuotaType(usageType); + quotaResourceStatementResponse.setUsageName(quotaType.getQuotaName()); + quotaResourceStatementResponse.setUnit(quotaType.getQuotaUnit()); + + quotaResourceStatementResponse.setQuotaUsageDetails(quotaUsageDetailsItems); + quotaResourceStatementResponse.setTotalQuotaUsed(totalQuotaUsed); + + return quotaResourceStatementResponse; + } + /** * Checks whether script variables are compatible with the usage type. First, we remove all script variables that correspond to the script's usage type variables. * Then, returns true if none of the remaining script variables match any usage types variables, and false otherwise. diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java index 81cb1011182d..c30780582d36 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java @@ -31,11 +31,11 @@ public class QuotaStatementResponse extends BaseResponse { @Param(description = "ID of the Account.") private String accountId; - @SerializedName(ApiConstants.ACCOUNT) + @SerializedName(ApiConstants.ACCOUNT_NAME) @Param(description = "Name of the Account.") private String accountName; - @SerializedName(ApiConstants.DOMAIN) + @SerializedName(ApiConstants.DOMAIN_ID) @Param(description = "ID of the Domain.") private String domainId; diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index 78acfc11682e..cc257c233def 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -28,9 +28,9 @@ public interface QuotaService extends PluggableService { - List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); + List getQuotaUsage(Long accountId, String accountName, List domainIds, Integer usageType, Date startDate, Date endDate); - List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); + List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate); void setLockAccount(Long accountId, Boolean state); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index a0ba2fbc751d..a788f0c8b01e 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -28,6 +28,7 @@ import com.cloud.projects.ProjectManager; import com.cloud.user.AccountService; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -37,6 +38,7 @@ import org.apache.cloudstack.api.command.QuotaEnabledCmd; import org.apache.cloudstack.api.command.QuotaListEmailConfigurationCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; +import org.apache.cloudstack.api.command.QuotaResourceStatementCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; @@ -58,17 +60,16 @@ import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.PermissionDeniedException; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.db.Filter; @Component public class QuotaServiceImpl extends ManagerBase implements QuotaService, Configurable, QuotaConfig { @@ -131,6 +132,7 @@ public List> getCommands() { cmdList.add(QuotaListEmailConfigurationCmd.class); cmdList.add(QuotaPresetVariablesListCmd.class); cmdList.add(QuotaValidateActivationRuleCmd.class); + cmdList.add(QuotaResourceStatementCmd.class); return cmdList; } @@ -152,76 +154,66 @@ public Boolean isQuotaServiceEnabled() { } @Override - public List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate) { - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } + public List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate) { + validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + + if (accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); } + Account account = _accountDao.findByIdIncludingRemoved(accountId); + Long domainId = account.getDomainId(); - startDate = startDate == null ? new Date() : startDate; + if (startDate == null && endDate == null) { + logger.debug("Retrieving last quota balance for {}.", account); + QuotaBalanceVO lastQuotaBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, null); - if (endDate == null) { - // adjust start date to end of day as there is no end date - startDate = _respBldr.startOfNextDay(startDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance1: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", on or before " + startDate); - } - List qbrecords = _quotaBalanceDao.lastQuotaBalanceVO(accountId, domainId, startDate); - if (logger.isDebugEnabled()) { - logger.debug("Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("Incorrect Date there are no quota records before this date " + startDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - if (startDate.before(endDate)) { - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance2: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", between " + startDate - + " and " + endDate); - } - List qbrecords = _quotaBalanceDao.findQuotaBalance(accountId, domainId, startDate, endDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance3: Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("There are no quota records between these dates start date " + startDate + " and end date:" + endDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); + if (lastQuotaBalance == null) { + logger.debug("Did not found a quota balance entry for {}.", account); + return new ArrayList<>(); } + + return List.of(lastQuotaBalance); + } + + if (endDate == null) { + endDate = DateUtils.addDays(new Date(), -1); + } + + List quotaBalances = _quotaBalanceDao.listQuotaBalances(accountId, domainId, startDate, endDate); + + if (quotaBalances.isEmpty()) { + logger.info("There are no quota balances for {} between [{}] and [{}].", account, + DateUtil.getOutputString(startDate), DateUtil.getOutputString(endDate)); + } + + return quotaBalances; + } + + protected void validateStartDateAndEndDateForListQuotaBalancesForAccount(Date startDate, Date endDate) { + if (startDate == null && endDate != null) { + throw new InvalidParameterValueException("Parameter \"enddate\" must be informed together with parameter \"startdate\"."); + } + + Date now = new Date(); + if (startDate != null && startDate.after(now)) { + throw new InvalidParameterValueException("The last balance can be at most from yesterday; therefore, the start date must be before today."); + } + + if (ObjectUtils.allNotNull(startDate, endDate) && startDate.after(endDate)) { + throw new InvalidParameterValueException("The start date cannot be after the end date."); } } @Override - public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { + public List getQuotaUsage(Long accountId, String accountName, List domainIds, Integer usageType, Date startDate, Date endDate) { if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } - logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", - usageType, accountId, domainId, startDate, endDate); + logger.debug("Getting quota records of type [{}] for account [{}] in domains [{}], between [{}] and [{}].", + usageType, accountId, domainIds, startDate, endDate); - return quotaUsageJoinDao.findQuotaUsage(accountId, domainId, usageType, null, null, null, startDate, endDate, null); + return quotaUsageJoinDao.findQuotaUsage(accountId, domainIds, usageType, null, null, null, startDate, endDate, null); } @Override diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java index adabc694f25a..cd0eeb2c3b48 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java @@ -16,18 +16,15 @@ // under the License. package org.apache.cloudstack.api.command; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import junit.framework.TestCase; @@ -36,31 +33,20 @@ public class QuotaBalanceCmdTest extends TestCase { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder quotaResponseBuilderMock; - @Test - public void testQuotaBalanceCmd() throws NoSuchFieldException, IllegalAccessException { - QuotaBalanceCmd cmd = new QuotaBalanceCmd(); - Field rbField = QuotaBalanceCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + @InjectMocks + @Spy + QuotaBalanceCmd quotaBalanceCmdSpy; - List quotaBalanceVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaBalance(Mockito.any(cmd.getClass()))).thenReturn(quotaBalanceVOList); - Mockito.when(responseBuilder.createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.when(responseBuilder.createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.lenient().when(responseBuilder.startOfNextDay(Mockito.any(Date.class))).thenReturn(new Date()); + @Test + public void executeTestSetResponseObject() { + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + Mockito.doReturn(expected).when(quotaResponseBuilderMock).createQuotaBalanceResponse(Mockito.eq(quotaBalanceCmdSpy)); - // end date not specified - cmd.setStartDate(new Date()); - cmd.setEndDate(null); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class)); - Mockito.verify(responseBuilder, Mockito.times(0)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + quotaBalanceCmdSpy.execute(); - // end date specified - cmd.setEndDate(new Date()); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + Assert.assertEquals(expected, quotaBalanceCmdSpy.getResponseObject()); } + } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index 81b4992082d9..37307db45629 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -38,13 +38,15 @@ import com.cloud.user.AccountManager; import com.cloud.user.UserVO; import com.cloud.utils.Pair; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; -import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; @@ -155,7 +157,7 @@ public class QuotaResponseBuilderImplTest extends TestCase { Date date = new Date(); @Mock - Account accountMock; + AccountVO accountMock; @Mock DomainVO domainVoMock; @@ -187,6 +189,9 @@ public class QuotaResponseBuilderImplTest extends TestCase { @Mock User callerUserMock; + @Mock + EntityManager entityManagerMock; + @Before public void setup() { CallContext.register(callerUserMock, callerAccountMock); @@ -249,7 +254,7 @@ public void testAddQuotaCredits() { credit.setCredit(new BigDecimal(amount)); Mockito.when(quotaCreditsDaoMock.saveCredits(Mockito.any(QuotaCreditsVO.class))).thenReturn(credit); - Mockito.when(quotaBalanceDaoMock.lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class))).thenReturn(new BigDecimal(111)); + Mockito.when(quotaBalanceDaoMock.getLastQuotaBalance(Mockito.anyLong(), Mockito.anyLong())).thenReturn(new BigDecimal(111)); Mockito.doReturn(userVoMock).when(quotaResponseBuilderSpy).getCreditorForQuotaCredits(credit); AccountVO account = new AccountVO(); @@ -295,33 +300,6 @@ public void testUpdateQuotaEmailTemplate() { assertTrue(quotaResponseBuilderSpy.updateQuotaEmailTemplate(cmd)); } - @Test - public void testCreateQuotaLastBalanceResponse() { - List quotaBalance = new ArrayList<>(); - // null balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(null, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // empty balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // valid balance test - QuotaBalanceVO entry = new QuotaBalanceVO(); - entry.setAccountId(2L); - entry.setCreditBalance(new BigDecimal(100)); - quotaBalance.add(entry); - quotaBalance.add(entry); - QuotaBalanceResponse resp = quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, null); - assertTrue(resp.getStartQuota().compareTo(new BigDecimal(200)) == 0); - } - @Test public void testStartOfNextDayWithoutParameters() { Date nextDate = quotaResponseBuilderSpy.startOfNextDay(); @@ -919,6 +897,46 @@ public void injectUsageTypeVariablesTestReturnInjectedVariables() { Assert.assertTrue(formattedVariables.containsValue("zonename")); } + private List getQuotaBalancesForTest() { + List balances = new ArrayList<>(); + + QuotaBalanceVO balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-10.42)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-18.94)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-29.37)); + balances.add(balance); + + return balances; + } + + @Test + public void createQuotaBalancesResponseTestCreateResponse() { + List balances = getQuotaBalancesForTest(); + + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + expected.setObjectName("balance"); + expected.setCurrency("$"); + + Mockito.doReturn(balances).when(quotaServiceMock).listQuotaBalancesForAccount(Mockito.any(), Mockito.any(), Mockito.any()); + QuotaBalanceResponse result = quotaResponseBuilderSpy.createQuotaBalanceResponse(new QuotaBalanceCmd()); + + Assert.assertEquals(expected.getCurrency(), result.getCurrency()); + + for (int i = 0; i < balances.size(); i++) { + Assert.assertEquals(balances.get(i).getUpdatedOn(), result.getBalances().get(i).getDate()); + Assert.assertEquals(balances.get(i).getCreditBalance(), result.getBalances().get(i).getBalance()); + } + } + @Test public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { List listUsage = new ArrayList<>(); @@ -1010,107 +1028,161 @@ public void setStatementItemResourcesTestDoNotShowResourcesDoNothing() { } @Test - public void getAccountIdForQuotaStatementTestLimitsToCallingAccountForNormalUser() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + public void getAccountIdForQuotaStatementTestReturnsProvidedAccount() { + long providedAccountId = 200L; - Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); - Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + Mockito.when(accountDaoMock.findByIdIncludingRemoved(providedAccountId)).thenReturn(accountMock); + Mockito.doNothing().when(accountManagerMock).checkAccess(callerAccountMock, null, false, accountMock); - try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { - callContextMocked.when(CallContext::current).thenReturn(callContextMock); + long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(providedAccountId, null); - Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); - - Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); - } + Assert.assertEquals(200L, result); + Mockito.verify(accountManagerMock).checkAccess(callerAccountMock, null, false, accountMock); } @Test - public void getAccountIdForQuotaStatementTestReturnsEntityOwnerIdWhenProvided() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminWithoutProvidedAccount() { + Mockito.when(callerAccountMock.getType()).thenReturn(Account.Type.ADMIN); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(-1L, null); - Mockito.doReturn(42L).when(cmd).getEntityOwnerId(); + assertNull(result); + Mockito.verify(accountManagerMock, Mockito.never()).getAccount(Mockito.anyLong()); + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsDomainAdminWithoutProvidedAccount() { + Mockito.when(callerAccountMock.getType()).thenReturn(Account.Type.DOMAIN_ADMIN); - Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(-1L, null); - Assert.assertEquals(Long.valueOf(42L), result); + assertNull(result); + Mockito.verify(accountManagerMock, Mockito.never()).getAccount(Mockito.anyLong()); } @Test - public void getAccountIdForQuotaStatementTestLimitsToCallingAccountWhenCallerIsAdminAndDomainIsNotProvided() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + public void getAccountIdForQuotaStatementTestReturnsFallbackAccountWhenNoAccountProvidedAndCallerIsNotAdmin() { + Mockito.when(callerAccountMock.getType()).thenReturn(Account.Type.NORMAL); - Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); - Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); - Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); - Mockito.doReturn(null).when(cmd).getDomainId(); + Mockito.when(accountDaoMock.findByIdIncludingRemoved(300L)).thenReturn(accountMock); + Mockito.doNothing().when(accountManagerMock).checkAccess(callerAccountMock, null, false, accountMock); - try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { - callContextMocked.when(CallContext::current).thenReturn(callContextMock); + long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(-1L, 300L); - Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + assertEquals(300L, result); + Mockito.verify(accountManagerMock).checkAccess(callerAccountMock, null, false, accountMock); + } - Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); - } + @Test + public void getAccountIdForQuotaStatementTestAccessDeniedForProvidedAccount() { + Mockito.when(accountDaoMock.findByIdIncludingRemoved(200L)).thenReturn(accountMock); + Mockito.doThrow(new PermissionDeniedException("Access denied")) + .when(accountManagerMock).checkAccess(callerAccountMock, null, false, accountMock); + + Assert.assertThrows(PermissionDeniedException.class, + () -> quotaResponseBuilderSpy.getAccountIdForQuotaStatement(200L, null)); + Mockito.verify(accountManagerMock).checkAccess(callerAccountMock, null, false, accountMock); } @Test - public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminAndDomainIsProvided() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + public void getDomainIdsForQuotaStatementTestReturnsNullPairWhenAccountIsProvided() { + Pair> result = quotaResponseBuilderSpy.getDomainIdsForQuotaStatement(100L, null, false); - Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); - Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); - Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); - Mockito.doReturn(10L).when(cmd).getDomainId(); + assertNull(result.first()); + assertNull(result.second()); + Mockito.verify(domainDaoMock, Mockito.never()).findByIdIncludingRemoved(Mockito.anyLong()); + } - try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { - callContextMocked.when(CallContext::current).thenReturn(callContextMock); + @Test + public void getDomainIdsForQuotaStatementTestReturnsProvidedDomainIdNonRecursively() { + Mockito.when(domainDaoMock.findByIdIncludingRemoved(5L)).thenReturn(domainVoMock); + Mockito.doNothing().when(accountManagerMock).checkAccess(callerAccountMock, domainVoMock); - Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + Pair> result = quotaResponseBuilderSpy.getDomainIdsForQuotaStatement(null, 5L, false); - Assert.assertNull(result); - } + assertEquals(5L, (long) result.first()); + assertEquals(List.of(5L), result.second()); + Mockito.verify(accountManagerMock).checkAccess(callerAccountMock, domainVoMock); } @Test - public void getDomainIdForQuotaStatementTestReturnsAccountDomainIdWhenAccountIdIsProvided() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); - AccountVO account = Mockito.mock(AccountVO.class); + public void getDomainIdsForQuotaStatementTestReturnsCallerDomainNonRecursively() { + Mockito.when(callerAccountMock.getDomainId()).thenReturn(7L); + + Pair> result = quotaResponseBuilderSpy.getDomainIdsForQuotaStatement(null, null, false); + + assertEquals(7L, (long) result.first()); + assertEquals(List.of(7L), result.second()); + Mockito.verify(domainDaoMock, Mockito.never()).findByIdIncludingRemoved(Mockito.anyLong()); + } - Mockito.doReturn(account).when(accountDaoMock).findByIdIncludingRemoved(55L); - Mockito.doReturn(77L).when(account).getDomainId(); + @Test + public void getDomainIdsForQuotaStatementTestReturnsProvidedDomainRecursively() { + List domainAndChildren = List.of(5L, 10L, 15L); + Mockito.when(domainDaoMock.findByIdIncludingRemoved(5L)).thenReturn(domainVoMock); + Mockito.when(domainDaoMock.getDomainAndChildrenIds(5L)).thenReturn(domainAndChildren); + Mockito.doNothing().when(accountManagerMock).checkAccess(callerAccountMock, domainVoMock); - Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, 55L); + Pair> result = quotaResponseBuilderSpy.getDomainIdsForQuotaStatement(null, 5L, true); - Assert.assertEquals(Long.valueOf(77L), result); + assertEquals(5L, (long) result.first()); + assertEquals(domainAndChildren, result.second()); + Mockito.verify(domainDaoMock).getDomainAndChildrenIds(5L); } @Test - public void getDomainIdForQuotaStatementTestReturnsProvidedDomainIdWhenAccountIdIsNull() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + public void getDomainIdsForQuotaStatementReturnsCallerDomainRecursively() { + List domainAndChildren = List.of(1L, 2L, 3L); + Mockito.when(callerAccountMock.getDomainId()).thenReturn(1L); - Mockito.doReturn(99L).when(cmd).getDomainId(); + Mockito.when(domainDaoMock.getDomainAndChildrenIds(1L)).thenReturn(domainAndChildren); - Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + Pair> result = quotaResponseBuilderSpy.getDomainIdsForQuotaStatement(null, null, true); - Assert.assertEquals(Long.valueOf(99L), result); + assertEquals(1L, (long) result.first()); + assertEquals(domainAndChildren, result.second()); + Mockito.verify(domainDaoMock).getDomainAndChildrenIds(1L); } @Test - public void getDomainIdForQuotaStatementTestFallsBackToCallingAccountDomainIdWhenNeitherAccountNorDomainIsProvided() { - QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); - Account account = Mockito.mock(Account.class); + public void getDomainIdsForQuotaStatementTestThrowsAccessDeniedForProvidedDomain() { + Mockito.when(domainDaoMock.findByIdIncludingRemoved(5L)).thenReturn(domainVoMock); + Mockito.doThrow(new PermissionDeniedException("Access denied")) + .when(accountManagerMock).checkAccess(callerAccountMock, domainVoMock); - Mockito.doReturn(null).when(cmd).getDomainId(); - Mockito.doReturn(123L).when(account).getDomainId(); - Mockito.doReturn(account).when(callContextMock).getCallingAccount(); + Assert.assertThrows(PermissionDeniedException.class, + () -> quotaResponseBuilderSpy.getDomainIdsForQuotaStatement(null, 5L, false)); + Mockito.verify(accountManagerMock).checkAccess(callerAccountMock, domainVoMock); + } - try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { - callContextMocked.when(CallContext::current).thenReturn(callContextMock); + @Test(expected = InvalidParameterValueException.class) + public void retrieveResourceTestThrowsExceptionForInvalidUsageType() { + Integer invalidUsageType = 999; + quotaResponseBuilderSpy.retrieveResource("validUuid", invalidUsageType); + } + + @Test + public void retrieveResourceTestReturnsNullForNonexistentResource() { + String invalidUuid = "nonexistentUuid"; + Integer validUsageType = QuotaTypes.ALLOCATED_VM; - Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + Mockito.doReturn(null).when(entityManagerMock).findByUuidIncludingRemoved(Mockito.any(), Mockito.eq(invalidUuid)); + InternalIdentity result = quotaResponseBuilderSpy.retrieveResource(invalidUuid, validUsageType); - Assert.assertEquals(123L, result.longValue()); - } + Assert.assertNull(result); + } + + @Test + public void retrieveResourceTestReturnsCorrectResource() { + String validUuid = "validUuid"; + Integer validUsageType = QuotaTypes.ALLOCATED_VM; + InternalIdentity mockResource = Mockito.mock(InternalIdentity.class); + + Mockito.doReturn(mockResource).when(entityManagerMock).findByUuidIncludingRemoved(Mockito.any(), Mockito.eq(validUuid)); + + InternalIdentity result = quotaResponseBuilderSpy.retrieveResource(validUuid, validUsageType); + + Assert.assertNotNull(result); + Assert.assertEquals(mockResource, result); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index a0fe63de8518..c0ee6c5fc3fe 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; @@ -31,7 +32,9 @@ import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.time.DateUtils; import org.joda.time.DateTime; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -43,7 +46,6 @@ import javax.naming.ConfigurationException; import java.lang.reflect.Field; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -51,6 +53,8 @@ @RunWith(MockitoJUnitRunner.class) public class QuotaServiceImplTest extends TestCase { + @Mock + AccountVO accountVoMock; @Mock AccountDao accountDaoMock; @Mock @@ -67,9 +71,6 @@ public class QuotaServiceImplTest extends TestCase { QuotaUsageJoinDao quotaUsageJoinDaoMock; @Mock QuotaResponseBuilder respBldr; - @Mock - private AccountVO accountVoMock; - @Spy @InjectMocks QuotaServiceImpl quotaServiceImplSpy; @@ -112,40 +113,16 @@ public void setup() throws IllegalAccessException, NoSuchFieldException, Configu quotaServiceImplSpy.configure("randomName", null); } - @Test - public void testFindQuotaBalanceVO() { - final long accountId = 2L; - final String accountName = "admin123"; - final long domainId = 1L; - final Date startDate = new DateTime().minusDays(2).toDate(); - final Date endDate = new Date(); - - List records = new ArrayList<>(); - QuotaBalanceVO qb = new QuotaBalanceVO(); - qb.setCreditBalance(new BigDecimal(100)); - qb.setAccountId(accountId); - records.add(qb); - - Mockito.when(respBldr.startOfNextDay(Mockito.any(Date.class))).thenReturn(startDate); - Mockito.when(quotaBalanceDao.findQuotaBalance(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(records); - Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); - - // with enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); - // without enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); - } - @Test public void testGetQuotaUsage() { final long accountId = 2L; final String accountName = "admin123"; - final long domainId = 1L; + final List domainIds = List.of(1L); final Date startDate = new DateTime().minusDays(2).toDate(); final Date endDate = new Date(); - quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); - Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), + quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainIds, QuotaTypes.IP_ADDRESS, startDate, endDate); + Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainIds), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); } @@ -182,4 +159,66 @@ public void testSetMinBalance() { Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsNullAndEndDateIsNotNullThrowsInvalidParameterException() { + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(null, new Date()); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterNowThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, null); + } + + @Test + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestEndDateIsAfterNowDoesNothing() { + Date startDate = DateUtils.addMinutes(new Date(), -1); + Date endDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterEndDateThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), -10); + Date endDate = DateUtils.addMinutes(new Date(), -15); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNullReturnsEmptyList() { + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(null).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNotNullReturnsIt() { + QuotaBalanceVO expected = new QuotaBalanceVO(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertEquals(expected, result.get(0)); + } + + @Test + public void listQuotaBalancesForAccountTestReturnsQuotaBalances() { + List expected = new ArrayList<>(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).listQuotaBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, new Date(), null); + + Assert.assertEquals(expected, result); + } + } diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java index ac840c00be32..3f2d85458d30 100644 --- a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java @@ -48,6 +48,8 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; +import javax.net.ssl.SSLContext; + import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustAllStrategy; import org.apache.http.entity.ContentType; @@ -97,7 +99,9 @@ protected boolean isValidJson(String json) { protected void setHttpClient() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { if (webhook.isSslVerification()) { - httpClient = HttpClients.createDefault(); + httpClient = HttpClients.custom() + .setSSLContext(SSLContext.getDefault()) + .build(); return; } httpClient = HttpClients diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index 061caf1573ff..000000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.23.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java deleted file mode 100644 index 42129944a194..000000000000 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -@Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject - private CapacityManager capacityManager; - - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::toString).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - - if (type == Host.Type.Storage) { - return suitableHosts; - } - String offeringHostTag = offering.getHostTag(); - - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostTag = ObjectUtils.allNotNull(offeringHostTag, templateTag) ? - String.format("%s, %s", offeringHostTag, templateTag) : - ObjectUtils.firstNonNull(offeringHostTag, templateTag); - logger.debug("Looking for hosts in dc [{}], pod [{}], cluster [{}] and complying with host tag(s): [{}]", dcId, podId, clusterId, hostTag); - } else { - logger.debug("Looking for hosts in dc: {} pod: {} cluster: {}", dcId , podId, clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); - - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); - return null; - } - - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } - - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { - if (suitableHosts.size() == returnUpTo) { - break; - } - if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Host %s is in avoid set, skipping this and trying other available hosts", host)); - } - continue; - } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Not using host %s; host has cpu capability? %s, host has capacity? %s", host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second())); - } - continue; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found a suitable host, adding to list: %s", host)); - } - suitableHosts.add(host); - } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } - return new ArrayList<>(); - } - return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } -} diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml deleted file mode 100644 index d84eaafaa5a7..000000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java deleted file mode 100644 index 538d7157184a..000000000000 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; - -@RunWith(MockitoJUnitRunner.class) -public class RandomAllocatorTest { - - @Mock - HostDao hostDao; - @InjectMocks - RandomAllocator randomAllocator; - - @Test - public void testListHostsByTags() { - Host.Type type = Host.Type.Routing; - Long id = 1L; - String templateTag = "tag1"; - String offeringTag = "tag2"; - HostVO host1 = Mockito.mock(HostVO.class); - HostVO host2 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); - - // No template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Different template tagged host - HostVO host3 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Matching template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - - // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); - - // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - } -} diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java index 3d79b9efdd13..4e44d8cb7359 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java @@ -41,15 +41,15 @@ public class HypervInvestigator extends AdapterBase implements Investigator { @Override public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM { - Status status = isAgentAlive(host); + Status status = getHostAgentStatus(host); if (status == null) { throw new UnknownVM(); } - return status == Status.Up ? true : null; + return status == Status.Up; } @Override - public Status isAgentAlive(Host agent) { + public Status getHostAgentStatus(Host agent) { if (agent.getHypervisorType() != Hypervisor.HypervisorType.Hyperv) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java index ce9fbe6c232c..da9a0d6e2919 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java @@ -19,10 +19,7 @@ package com.cloud.ha; import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.host.Host; -import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; @@ -34,11 +31,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.ha.HAManager; +import org.apache.cloudstack.kvm.ha.KVMHostActivityChecker; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import javax.inject.Inject; -import java.util.Arrays; +import java.util.Collections; import java.util.List; public class KVMInvestigator extends AdapterBase implements Investigator { @@ -54,13 +52,15 @@ public class KVMInvestigator extends AdapterBase implements Investigator { private HAManager haManager; @Inject private DataStoreProviderManager dataStoreProviderMgr; + @Inject + private KVMHostActivityChecker hostActivityChecker; @Override public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM { if (haManager.isHAEligible(host)) { return haManager.isVMAliveOnHost(host); } - Status status = isAgentAlive(host); + Status status = getHostAgentStatus(host); logger.debug("HA: HOST is ineligible legacy state {} for host {}", status, host); if (status == null) { throw new UnknownVM(); @@ -73,86 +73,41 @@ public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws Unkno } @Override - public Status isAgentAlive(Host agent) { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + public Status getHostAgentStatus(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { return null; } - if (haManager.isHAEligible(agent)) { - return haManager.getHostStatus(agent); + if (haManager.isHAEligible(host)) { + return haManager.getHostStatusFromHAConfig(host); } - List clusterPools = _storagePoolDao.findPoolsInClusters(Arrays.asList(agent.getClusterId()), null); - boolean storageSupportHA = storageSupportHa(clusterPools); - if (!storageSupportHA) { - List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType()); - storageSupportHA = storageSupportHa(zonePools); + List clusterPools = _storagePoolDao.findPoolsInClusters(Collections.singletonList(host.getClusterId()), null); + boolean storageSupportsHA = storageSupportsHA(clusterPools); + if (!storageSupportsHA) { + List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), host.getHypervisorType()); + storageSupportsHA = storageSupportsHA(zonePools); } - if (!storageSupportHA) { - logger.warn("Agent investigation was requested on host {}, but host does not support investigation because it has no NFS storage. Skipping investigation.", agent); + if (!storageSupportsHA) { + logger.warn("Agent investigation was requested on host {}, but host does not support investigation" + + " because it has no HA supported storage. Skipping investigation.", host); return null; } - Status hostStatus = null; - Status neighbourStatus = null; - boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); - CheckOnHostCommand cmd = new CheckOnHostCommand(agent, reportFailureIfOneStorageIsDown); - - try { - Answer answer = _agentMgr.easySend(agent.getId(), cmd); - if (answer != null) { - hostStatus = answer.getResult() ? Status.Down : Status.Up; - } - } catch (Exception e) { - logger.debug("Failed to send command to host: {}", agent); - } - if (hostStatus == null) { - hostStatus = Status.Disconnected; - } - - List neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); - for (HostVO neighbor : neighbors) { - if (neighbor.getId() == agent.getId() - || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { - continue; - } - logger.debug("Investigating host:{} via neighbouring host:{}", agent, neighbor); - try { - Answer answer = _agentMgr.easySend(neighbor.getId(), cmd); - if (answer != null) { - neighbourStatus = answer.getResult() ? Status.Down : Status.Up; - logger.debug("Neighbouring host:{} returned status:{} for the investigated host:{}", neighbor, neighbourStatus, agent); - if (neighbourStatus == Status.Up) { - break; - } - } - } catch (Exception e) { - logger.debug("Failed to send command to host: {}", neighbor); - } - } - if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Disconnected; - } - if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Down; - } - logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, agent); - return hostStatus; + return hostActivityChecker.getHostAgentStatus(host); } - private boolean storageSupportHa(List pools) { - boolean storageSupportHA = false; + private boolean storageSupportsHA(List pools) { for (StoragePoolVO pool : pools) { DataStoreProvider storeProvider = dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName()); DataStoreDriver storeDriver = storeProvider.getDataStoreDriver(); if (storeDriver instanceof PrimaryDataStoreDriver) { PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver; if (primaryStoreDriver.isStorageSupportHA(pool.getPoolType())) { - storageSupportHA = true; - break; + return true; } } } - return storageSupportHA; + return false; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java new file mode 100644 index 000000000000..2e9852f7bc1e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java @@ -0,0 +1,123 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Communicates with the cloudstack-image-server control socket via socat. + * + * Protocol: newline-delimited JSON over a Unix domain socket. + * Actions: register, unregister, status. + */ +public class ImageServerControlSocket { + private static final Logger LOGGER = LogManager.getLogger(ImageServerControlSocket.class); + static final String CONTROL_SOCKET_PATH = "/var/run/cloudstack/image-server.sock"; + private static final Gson GSON = new GsonBuilder().create(); + + private ImageServerControlSocket() { + } + + /** + * Send a JSON message to the image server control socket and return the + * parsed response, or null on communication failure. + */ + static JsonObject sendMessage(Map message) { + String json = GSON.toJson(message); + Script script = new Script("/bin/bash", LOGGER); + script.add("-c"); + script.add(String.format("echo '%s' | socat -t5 - UNIX-CONNECT:%s", + json.replace("'", "'\\''"), CONTROL_SOCKET_PATH)); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = script.execute(parser); + if (result != null) { + LOGGER.error("Control socket communication failed: {}", result); + return null; + } + String output = parser.getLines(); + if (output == null || output.trim().isEmpty()) { + LOGGER.error("Empty response from control socket"); + return null; + } + try { + return JsonParser.parseString(output.trim()).getAsJsonObject(); + } catch (Exception e) { + LOGGER.error("Failed to parse control socket response: {}", output, e); + return null; + } + } + + /** + * Register a transfer config with the image server. + * @return true if the server accepted the registration. + */ + public static boolean registerTransfer(String transferId, Map config) { + Map msg = new HashMap<>(); + msg.put("action", "register"); + msg.put("transfer_id", transferId); + msg.put("config", config); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } + + /** + * Unregister a transfer from the image server. + * @return the number of remaining active transfers, or -1 on error. + */ + public static int unregisterTransfer(String transferId) { + Map msg = new HashMap<>(); + msg.put("action", "unregister"); + msg.put("transfer_id", transferId); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return -1; + } + if (!"ok".equals(resp.has("status") ? resp.get("status").getAsString() : null)) { + return -1; + } + return resp.has("active_transfers") ? resp.get("active_transfers").getAsInt() : -1; + } + + /** + * Check whether the image server control socket is responsive. + * @return true if the server responded with status "ok". + */ + public static boolean isReady() { + Map msg = new HashMap<>(); + msg.put("action", "status"); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index e9a7ac8951ce..f30e2c779195 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -35,10 +35,9 @@ public class KVMHABase { protected Logger logger = LogManager.getLogger(getClass()); private long _timeout = 60000; /* 1 minutes */ - protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); - protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + protected long _heartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); - protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); + protected long _heartBeatUpdateRetrySleepInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); public static enum PoolType { PrimaryStorage, SecondaryStorage @@ -138,7 +137,7 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) { /* Can't find the mount point? */ /* we need to mount it under poolName */ if (poolName != null) { - Script mount = new Script("/bin/bash", 60000); + Script mount = new Script("/bin/bash", _timeout); mount.add("-c"); mount.add("mount " + mountSource + " " + destPath); String result = mount.execute(); @@ -154,7 +153,6 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) { } protected String getMountPoint(HAStoragePool storagePool) { - StoragePool pool = null; String poolName = null; try { @@ -171,7 +169,6 @@ protected String getMountPoint(HAStoragePool storagePool) { } poolName = pool.getName(); } - } catch (LibvirtException e) { logger.debug("Ignoring libvirt error.", e); } finally { @@ -234,7 +231,7 @@ protected String runScriptRetry(String cmdString, OutputInterpreter interpreter) return result; } - public Boolean checkingHeartBeat() { + public Boolean hasHeartBeat() { // TODO Auto-generated method stub return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index db6190fa8f28..0ee59c95da39 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -26,44 +26,43 @@ public class KVMHAChecker extends KVMHABase implements Callable { private List storagePools; private HostTO host; - private boolean reportFailureIfOneStorageIsDown; + private boolean reportIfHeartBeatFailedForOneStoragePool; - public KVMHAChecker(List pools, HostTO host, boolean reportFailureIfOneStorageIsDown) { + public KVMHAChecker(List pools, HostTO host, boolean reportIfHeartBeatFailedForOneStoragePool) { this.storagePools = pools; this.host = host; - this.reportFailureIfOneStorageIsDown = reportFailureIfOneStorageIsDown; + this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool; } /* - * True means heartbeaing is on going, or we can't get it's status. False - * means heartbeating is stopped definitely + * True means heart beating is on going, or we can't get it's status. + * False means heart beating is stopped definitely. */ @Override - public Boolean checkingHeartBeat() { - boolean validResult = false; - - String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); - - logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools)); + public Boolean hasHeartBeat() { + String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), + storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); + logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools); + boolean heartBeatCheckResult = false; for (HAStoragePool pool : storagePools) { - validResult = pool.getPool().checkingHeartBeat(pool, host); - if (reportFailureIfOneStorageIsDown && !validResult) { + heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host); + if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) { break; } } - if (!validResult) { - logger.warn(String.format("All checks with KVMHAChecker for %s considered it as dead. It may cause a shutdown of the host.", hostAndPools)); + if (!heartBeatCheckResult) { + logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools); } - return validResult; + return heartBeatCheckResult; } @Override public Boolean call() throws Exception { // logger.addAppender(new org.apache.log4j.ConsoleAppender(new // org.apache.log4j.PatternLayout(), "System.out")); - return checkingHeartBeat(); + return hasHeartBeat(); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index aa868ff1d3f2..9f1b849e9727 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -34,53 +34,49 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { - private final Map storagePool = new ConcurrentHashMap<>(); + private final Map haStoragePools = new ConcurrentHashMap<>(); private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; - public KVMHAMonitor(HAStoragePool pool, String host) { - if (pool != null) { - storagePool.put(pool.getPoolUUID(), pool); - } + public KVMHAMonitor(String host) { hostPrivateIp = host; - rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } public void addStoragePool(HAStoragePool pool) { - synchronized (storagePool) { - storagePool.put(pool.getPoolUUID(), pool); + synchronized (haStoragePools) { + haStoragePools.put(pool.getPoolUUID(), pool); } } public void removeStoragePool(String uuid) { - synchronized (storagePool) { - HAStoragePool pool = storagePool.get(uuid); + synchronized (haStoragePools) { + HAStoragePool pool = haStoragePools.get(uuid); if (pool != null) { Script.runSimpleBashScript("umount " + pool.getMountDestPath()); - storagePool.remove(uuid); + haStoragePools.remove(uuid); } } } public List getStoragePools() { - synchronized (storagePool) { - return new ArrayList<>(storagePool.values()); + synchronized (haStoragePools) { + return new ArrayList<>(haStoragePools.values()); } } public HAStoragePool getStoragePool(String uuid) { - synchronized (storagePool) { - return storagePool.get(uuid); + synchronized (haStoragePools) { + return haStoragePools.get(uuid); } } protected void runHeartBeat() { - synchronized (storagePool) { + synchronized (haStoragePools) { Set removedPools = new HashSet<>(); - for (String uuid : storagePool.keySet()) { - HAStoragePool primaryStoragePool = storagePool.get(uuid); + for (String uuid : haStoragePools.keySet()) { + HAStoragePool primaryStoragePool = haStoragePools.get(uuid); if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(primaryStoragePool.getPool().getType())) { checkForNotExistingLibvirtStoragePools(removedPools, uuid); if (removedPools.contains(uuid)) { @@ -91,7 +87,7 @@ protected void runHeartBeat() { result = executePoolHeartBeatCommand(uuid, primaryStoragePool, result); if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + logger.warn("Write heartbeat for pool [{}] failed: {}; stopping cloudstack-agent.", uuid, result); primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false);; } } @@ -104,20 +100,18 @@ protected void runHeartBeat() { } private String executePoolHeartBeatCommand(String uuid, HAStoragePool primaryStoragePool, String result) { - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + for (int attempt = 1; attempt <= _heartBeatUpdateMaxTries; attempt++) { result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - - if (result != null) { - logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { + if (result == null) { break; } + logger.warn("Write heartbeat for pool [{}] failed: {}; try: {} of {}.", uuid, result, attempt, _heartBeatUpdateMaxTries); + try { + Thread.sleep(_heartBeatUpdateRetrySleepInMs); + } catch (InterruptedException e) { + logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } } return result; } @@ -128,21 +122,21 @@ private void checkForNotExistingLibvirtStoragePools(Set removedPools, St StoragePool storage = conn.storagePoolLookupByUUIDString(uuid); if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { if (storage == null) { - logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + logger.debug("Libvirt storage pool [{}] not found, removing from HA list.", uuid); } else { - logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + logger.debug("Libvirt storage pool [{}] found, but not running, removing from HA list.", uuid); } removedPools.add(uuid); } - logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { - logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); if (e.toString().contains("pool not found")) { - logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + logger.debug("Removing pool [{}] from HA monitor since it was deleted.", uuid); removedPools.add(uuid); } } @@ -155,11 +149,10 @@ public void run() { runHeartBeat(); try { - Thread.sleep(_heartBeatUpdateFreq); + Thread.sleep(_heartBeatUpdateFreqInMs); } catch (InterruptedException e) { logger.debug("[IGNORED] Interrupted between heartbeats.", e); } } } - } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index e6937b515e95..c13be64a3a3e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -39,12 +39,12 @@ public KVMHAVMActivityChecker(final HAStoragePool pool, final HostTO host, final } @Override - public Boolean checkingHeartBeat() { - return this.storagePool.getPool().vmActivityCheck(storagePool, host, activityScriptTimeout, volumeUuidList, vmActivityCheckPath, suspectTimeInSeconds); + public Boolean hasHeartBeat() { + return this.storagePool.getPool().hasVmActivity(storagePool, host, activityScriptTimeout, volumeUuidList, vmActivityCheckPath, suspectTimeInSeconds); } @Override public Boolean call() throws Exception { - return checkingHeartBeat(); + return hasHeartBeat(); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0f0f80c0c17d..d0a162e0f6ff 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -124,6 +124,7 @@ import org.libvirt.LibvirtException; import org.libvirt.MemoryStatistic; import org.libvirt.Network; +import org.libvirt.SchedLongParameter; import org.libvirt.SchedParameter; import org.libvirt.SchedUlongParameter; import org.libvirt.Secret; @@ -386,6 +387,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s --metadata"; + public static final int IMAGE_SERVER_DEFAULT_PORT = 54322; + public static final String IMAGE_SERVER_SYSTEMD_UNIT_NAME = "cloudstack-image-server"; + protected int qcow2DeltaMergeTimeout; private String modifyVlanPath; @@ -399,6 +403,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String heartBeatPath; private String vmActivityCheckPath; private String nasBackupPath; + private String imageServerPath; + private boolean imageServerTlsEnabled = false; + private String imageServerListenAddress; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -813,6 +820,18 @@ public String getNasBackupPath() { return nasBackupPath; } + public String getImageServerPath() { + return imageServerPath; + } + + public boolean isImageServerTlsEnabled() { + return imageServerTlsEnabled; + } + + public String getImageServerListenAddress() { + return imageServerListenAddress; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -883,6 +902,25 @@ protected enum HealthCheckResult { SUCCESS, FAILURE, IGNORE } + public enum CpuSchedulerParameter { + CPU_SHARES("cpu_shares"), PERIOD("vcpu_period"), QUOTA("vcpu_quota"); + + private String name; + + CpuSchedulerParameter(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return getName(); + } + } + protected BridgeType bridgeType; protected StorageSubsystemCommandHandler storageHandler; @@ -1057,6 +1095,9 @@ public boolean configure(final String name, final Map params) th cachePath = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_CACHE_LOCATION); + imageServerTlsEnabled = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_ENABLED); + imageServerListenAddress = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_LISTEN_ADDRESS); + params.put("domr.scripts.dir", domrScriptsDir); virtRouterResource = new VirtualRoutingResource(this); @@ -1120,6 +1161,12 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find nasbackup.sh"); } + String imageServerMain = Script.findScript(kvmScriptsDir, "imageserver/__main__.py"); + if (imageServerMain == null) { + throw new ConfigurationException("Unable to find imageserver package"); + } + imageServerPath = new File(imageServerMain).getParent(); + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); @@ -1388,9 +1435,9 @@ public boolean configure(final String name, final Map params) th final String[] info = NetUtils.getNetworkParams(privateNic); - kvmhaMonitor = new KVMHAMonitor(null, info[0]); - final Thread ha = new Thread(kvmhaMonitor); - ha.start(); + kvmhaMonitor = new KVMHAMonitor(info[0]); + final Thread haMonitorThread = new Thread(kvmhaMonitor); + haMonitorThread.start(); storagePoolManager = new KVMStoragePoolManager(storageLayer, kvmhaMonitor); @@ -2965,23 +3012,61 @@ protected String getUuid(String uuid) { protected void setQuotaAndPeriod(VirtualMachineTO vmTO, CpuTuneDef ctd) { if (vmTO.isLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) { Double cpuQuotaPercentage = vmTO.getCpuQuotaPercentage(); - int period = CpuTuneDef.DEFAULT_PERIOD; - int quota = (int) (period * cpuQuotaPercentage); - if (quota < CpuTuneDef.MIN_QUOTA) { - LOGGER.info("Calculated quota (" + quota + ") below the minimum (" + CpuTuneDef.MIN_QUOTA + ") for VM domain " + vmTO.getUuid() + ", setting it to minimum " + - "and calculating period instead of using the default"); - quota = CpuTuneDef.MIN_QUOTA; - period = (int) ((double) quota / cpuQuotaPercentage); - if (period > CpuTuneDef.MAX_PERIOD) { - LOGGER.info("Calculated period (" + period + ") exceeds the maximum (" + CpuTuneDef.MAX_PERIOD + - "), setting it to the maximum"); - period = CpuTuneDef.MAX_PERIOD; - } + Pair periodAndQuota = getPeriodAndQuota(cpuQuotaPercentage); + ctd.setPeriod(periodAndQuota.first()); + ctd.setQuota(periodAndQuota.second()); + LOGGER.info("Setting quota = [{}] and period = [{}] to VM domain [{}].", periodAndQuota.second(), periodAndQuota.first(), vmTO.getUuid()); + } + } + + /** + * Calculates the CPU period and quota based on the quota percentage defined by the Management Server + * @param cpuQuotaPercentage CPU quota percentage defined by the Management Server + * @return The period and quota to be defined for the VM's domain + */ + protected Pair getPeriodAndQuota(double cpuQuotaPercentage) { + int period = CpuTuneDef.DEFAULT_PERIOD; + long quota = (long) (period * cpuQuotaPercentage); + if (quota < CpuTuneDef.MIN_QUOTA) { + LOGGER.info("Calculated quota ({}) below the minimum ({}), setting it to minimum and calculating period instead of using the default", quota, CpuTuneDef.MIN_QUOTA); + quota = CpuTuneDef.MIN_QUOTA; + period = (int) ((double) quota / cpuQuotaPercentage); + if (period > CpuTuneDef.MAX_PERIOD) { + LOGGER.info("Calculated period ({}) exceeds the maximum ({}), setting it to the maximum", period, CpuTuneDef.MAX_PERIOD); + period = CpuTuneDef.MAX_PERIOD; } - ctd.setQuota(quota); - ctd.setPeriod(period); - LOGGER.info("Setting quota=" + quota + ", period=" + period + " to VM domain " + vmTO.getUuid()); } + + LOGGER.info("Calculated period = [{}] and quota = [{}] given the [{}] quota percentage.", period, quota, cpuQuotaPercentage); + return new Pair<>(period, quota); + } + + /** + * Dynamically updates the domain's "vcpu_quota" and "period" fields of the CPU tune definition. + * This is required because the values of the fields must change according to the new CPU speed of the VM. + * When the CPU limitation is removed from the domain, the "vcpu_quota" field is set to 17,592,186,044,415. + * @param domain VM's domain. + * @param vmTO VM's transfer object, which contains the required fields to update the "vcpu_quota" and "period" fields. + * @param limitCpuUseChange Indicates whether the CPU limitation for the VM has changed. + * @throws org.libvirt.LibvirtException + **/ + public void updateCpuQuotaAndPeriod(Domain domain, VirtualMachineTO vmTO, boolean limitCpuUseChange) throws LibvirtException { + if (hypervisorLibvirtVersion < MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE || (!limitCpuUseChange && !vmTO.isLimitCpuUse())) { + logger.info("Not updating the [{}] and [{}] for the [{}] domain, because [{}].", + CpuSchedulerParameter.QUOTA, CpuSchedulerParameter.PERIOD, domain.getName(), hypervisorLibvirtVersion < MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE ? + "the current Libvirt version does not support CPU tune" : "it was not requested to remove, change or apply CPU limitation for the instance."); + return; + } + + if (limitCpuUseChange && !vmTO.isLimitCpuUse()) { + logger.info("Updating the [{}] of the [{}] domain to [{}], because CPU limitation has been removed.", CpuSchedulerParameter.QUOTA, domain.getName(), CpuTuneDef.MAX_CPU_QUOTA); + LibvirtComputingResource.setQuota(domain, CpuTuneDef.MAX_CPU_QUOTA); + return; + } + + Pair periodAndQuota = getPeriodAndQuota(vmTO.getCpuQuotaPercentage()); + LibvirtComputingResource.setPeriod(domain, periodAndQuota.first()); + LibvirtComputingResource.setQuota(domain, periodAndQuota.second()); } protected void enlightenWindowsVm(VirtualMachineTO vmTO, FeaturesDef features) { @@ -3445,10 +3530,10 @@ protected GuestResourceDef createGuestResourceDef(VirtualMachineTO vmTO){ grd.setMemBalloning(!noMemBalloon); - Long maxRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMaxRam()); - - grd.setMemorySize(maxRam); - grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, maxRam)); + long requestedRam = ByteScaleUtils.bytesToKibibytes(vmTO.getRequestedRam()); + long minRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMinRam()); + grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, requestedRam, minRam)); + grd.setMaxMemory(ByteScaleUtils.bytesToKibibytes(vmTO.getMaxRam())); int vcpus = vmTO.getCpus(); Integer maxVcpus = vmTO.getVcpuMaxLimit(); @@ -3459,18 +3544,19 @@ protected GuestResourceDef createGuestResourceDef(VirtualMachineTO vmTO){ return grd; } - protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long maxRam) { - long retVal = maxRam; + protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long requestedRam, long minRam) { if (noMemBalloon) { - LOGGER.warn(String.format("Setting VM's [%s] current memory as max memory [%s] due to memory ballooning is disabled. If you are using a custom service offering, verify if memory ballooning really should be disabled.", vmTO.toString(), maxRam)); - } else if (vmTO != null && vmTO.getType() != VirtualMachine.Type.User) { - LOGGER.warn(String.format("Setting System VM's [%s] current memory as max memory [%s].", vmTO.toString(), maxRam)); - } else { - long minRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMinRam()); - LOGGER.debug(String.format("Setting VM's [%s] current memory as min memory [%s] due to memory ballooning is enabled.", vmTO.toString(), minRam)); - retVal = minRam; + LOGGER.warn("Setting VM's [{}] current memory as requested memory [{}] due to memory ballooning is disabled.", vmTO.toString(), requestedRam); + return requestedRam; } - return retVal; + + if (vmTO != null && vmTO.getType() != VirtualMachine.Type.User) { + LOGGER.warn("Setting System VM's [{}] current memory as requested memory [{}].", vmTO.toString(), requestedRam); + return requestedRam; + } + + LOGGER.debug("Setting VM's [{}] current memory as min memory [{}] due to memory ballooning is enabled.", vmTO.toString(), minRam); + return minRam; } /** @@ -5290,6 +5376,24 @@ public void removeCheckpointsOnVm(String vmName, String volumeUuid, List logger.debug("Removed all checkpoints of volume [{}] on VM [{}].", volumeUuid, vmName); } + public Map getDiskPathLabelMap(String vmName) { + try { + Connect conn = LibvirtConnection.getConnectionByVmName(vmName); + List disks = getDisks(conn, vmName); + Map diskPathLabelMap = new HashMap<>(); + for (DiskDef disk : disks) { + if (disk.getDeviceType() != DeviceType.DISK) { + continue; + } + diskPathLabelMap.put(disk.getDiskPath(), disk.getDiskLabel()); + } + return diskPathLabelMap; + } catch (LibvirtException e) { + logger.error("Failed to get disk path label map for VM [{}] due to: [{}].", vmName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + public boolean recreateCheckpointsOnVm(List volumes, String vmName, Connect conn) { logger.debug("Trying to recreate checkpoints on VM [{}] with volumes [{}].", vmName, volumes); try { @@ -6255,29 +6359,59 @@ public static long countDomainRunningVcpus(Domain dm) throws LibvirtException { **/ public static Integer getCpuShares(Domain dm) throws LibvirtException { for (SchedParameter c : dm.getSchedulerParameters()) { - if (c.field.equals("cpu_shares")) { + if (c.field.equals(CpuSchedulerParameter.CPU_SHARES.getName())) { return Integer.parseInt(c.getValueAsString()); } } - LOGGER.warn(String.format("Could not get cpu_shares of domain: [%s]. Returning default value of 0. ", dm.getName())); + LOGGER.warn("Could not get [{}] of domain: [{}]. Returning default value of 0. ", CpuSchedulerParameter.CPU_SHARES.getName(), dm.getName()); return 0; } /** - * Sets the cpu_shares (priority) of the running VM
    + * Updates the cpu_shares (priority) of the running VM. * @param dm domain of the VM. * @param cpuShares new priority of the running VM. - * @throws org.libvirt.LibvirtException **/ public static void setCpuShares(Domain dm, Integer cpuShares) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.CPU_SHARES.getName(), dm.getName(), cpuShares); SchedUlongParameter[] params = new SchedUlongParameter[1]; params[0] = new SchedUlongParameter(); - params[0].field = "cpu_shares"; + params[0].field = CpuSchedulerParameter.CPU_SHARES.getName(); params[0].value = cpuShares; dm.setSchedulerParameters(params); } + /** + * Updates the period of the running VM. + * @param domain domain of the VM. + * @param period new period of the running VM. + **/ + public static void setPeriod(Domain domain, int period) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.PERIOD.getName(), domain.getName(), period); + SchedUlongParameter[] params = new SchedUlongParameter[1]; + params[0] = new SchedUlongParameter(); + params[0].field = CpuSchedulerParameter.PERIOD.getName(); + params[0].value = period; + + domain.setSchedulerParameters(params); + } + + /** + * Updates the quota of the running VM. + * @param domain domain of the VM. + * @param quota new quota of the running VM. + **/ + public static void setQuota(Domain domain, long quota) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.QUOTA.getName(), domain.getName(), quota); + SchedLongParameter[] params = new SchedLongParameter[1]; + params[0] = new SchedLongParameter(); + params[0].field = CpuSchedulerParameter.QUOTA.getName(); + params[0].value = quota; + + domain.setSchedulerParameters(params); + } + /** * Set up a libvirt secret for a volume. If Libvirt says that a secret already exists for this volume path, we use its uuid. * The UUID of the secret needs to be prescriptive such that we can register the same UUID on target host during live migration diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index 4d823783a99a..e114669b8b56 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -542,7 +542,7 @@ private void extractCpuTuneDef(final Element rootElement) { final String quota = getTagValue("quota", cpuTuneDefElement); if (StringUtils.isNotBlank(quota)) { - cpuTuneDef.setQuota((Integer.parseInt(quota))); + cpuTuneDef.setQuota((Long.parseLong(quota))); } final String period = getTagValue("period", cpuTuneDefElement); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 097a9b8dd322..bf8b1af6c18d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -443,15 +443,15 @@ public String toString() { } public static class GuestResourceDef { - private long memory; + private long maxMemory; private long currentMemory = -1; private int vcpu = -1; private int maxVcpu = -1; private boolean memoryBalloning = false; private int memoryBalloonStatsPeriod = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_MEMBALLOON_STATS_PERIOD); - public void setMemorySize(long mem) { - this.memory = mem; + public void setMaxMemory(long mem) { + this.maxMemory = mem; } public void setCurrentMem(long currMem) { @@ -484,8 +484,8 @@ public String toString() { response.append(String.format("%s\n", this.currentMemory)); response.append(String.format("%s\n", this.currentMemory)); - if (this.memory > this.currentMemory) { - response.append(String.format("%s\n", this.memory)); + if (this.maxMemory > this.currentMemory) { + response.append(String.format("%s\n", this.maxMemory)); response.append(String.format(" \n", this.maxVcpu - 1, this.currentMemory)); } @@ -1920,11 +1920,12 @@ public String toString() { public static class CpuTuneDef { private int _shares = 0; - private int quota = 0; + private long quota = 0; private int period = 0; static final int DEFAULT_PERIOD = 10000; static final int MIN_QUOTA = 1000; static final int MAX_PERIOD = 1000000; + public static final long MAX_CPU_QUOTA = 17592186044415L; public void setShares(int shares) { _shares = shares; @@ -1934,11 +1935,11 @@ public int getShares() { return _shares; } - public int getQuota() { + public long getQuota() { return quota; } - public void setQuota(int quota) { + public void setQuota(long quota) { this.quota = quota; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java index 48996a7ba97c..f901fd97ca76 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java @@ -26,6 +26,7 @@ import java.util.concurrent.Future; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckOnHostAnswer; import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.agent.api.to.HostTO; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; @@ -45,20 +46,21 @@ public Answer execute(final CheckOnHostCommand command, final LibvirtComputingRe final List pools = monitor.getStoragePools(); final HostTO host = command.getHost(); - final KVMHAChecker ha = new KVMHAChecker(pools, host, command.isCheckFailedOnOneStorage()); + + final KVMHAChecker ha = new KVMHAChecker(pools, host, command.shouldReportIfHeartBeatFailedForOneStoragePool()); final Future future = executors.submit(ha); try { - final Boolean result = future.get(); - if (result) { - return new Answer(command, false, "Heart is beating..."); + final Boolean hasHeartBeat = future.get(); + if (hasHeartBeat) { + return new CheckOnHostAnswer(command, true, "Heart is beating"); } else { - return new Answer(command); + return new CheckOnHostAnswer(command, "Heart is not beating"); } } catch (final InterruptedException e) { - return new Answer(command, false, "CheckOnHostCommand: can't get status of host: InterruptedException"); + return new CheckOnHostAnswer(command, "CheckOnHostCommand: can't get status of host: InterruptedException"); } catch (final ExecutionException e) { - return new Answer(command, false, "CheckOnHostCommand: can't get status of host: ExecutionException"); + return new CheckOnHostAnswer(command, "CheckOnHostCommand: can't get status of host: ExecutionException"); } } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index d3f537dc9173..b132f6d98ce9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -49,8 +49,8 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid()); if (primaryPool.isPoolSupportHA()) { - final HAStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid()); + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java new file mode 100644 index 000000000000..7cf05da9b211 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -0,0 +1,178 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.storage.resource.IpTablesHelper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = CreateImageTransferCommand.class) +public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private static final String IMAGE_SERVER_TLS_CERT_FILE = "/etc/cloudstack/agent/cloud.crt"; + private static final String IMAGE_SERVER_TLS_KEY_FILE = "/etc/cloudstack/agent/cloud.key"; + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private static String shellQuote(String value) { + return "'" + value.replace("'", "'\\''") + "'"; + } + + private boolean startImageServerIfNotRunning(int imageServerPort, String listenAddress, LibvirtComputingResource resource) { + final String imageServerPackageDir = resource.getImageServerPath(); + final String imageServerParentDir = new File(imageServerPackageDir).getParent(); + final String imageServerModuleName = new File(imageServerPackageDir).getName(); + final boolean tlsEnabled = resource.isImageServerTlsEnabled(); + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null && ImageServerControlSocket.isReady()) { + return true; + } + + resetService(unitName); + if (checkResult != null) { + StringBuilder systemdRunCmd = new StringBuilder(String.format( + "systemd-run --unit=%s --property=Restart=no --property=WorkingDirectory=%s /usr/bin/python3 -m %s --listen %s --port %d", + unitName, shellQuote(imageServerParentDir), imageServerModuleName, shellQuote(listenAddress), imageServerPort)); + + if (tlsEnabled) { + systemdRunCmd.append(" --tls-enabled"); + systemdRunCmd.append(" --tls-cert-file ").append(IMAGE_SERVER_TLS_CERT_FILE); + systemdRunCmd.append(" --tls-key-file ").append(IMAGE_SERVER_TLS_KEY_FILE); + } + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd.toString()); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start the Image server: %s", startResult)); + return false; + } + } + + int maxWaitSeconds = 10; + int pollIntervalMs = 1000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serverReady = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + if (ImageServerControlSocket.isReady()) { + serverReady = true; + logger.info(String.format("Image server control socket is ready (attempt %d)", attempt + 1)); + break; + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + if (!serverReady) { + logger.error(String.format("Image server control socket not ready within %d seconds", maxWaitSeconds)); + return false; + } + + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + String.format("Error in opening up image server port %d", imageServerPort)); + + return true; + } + + public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + ImageTransfer.Backend backend = cmd.getBackend(); + + if (StringUtils.isBlank(transferId)) { + return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); + } + + final Map payload = new HashMap<>(); + payload.put("backend", backend.toString()); + payload.put("idle_timeout_seconds", cmd.getIdleTimeoutSeconds()); + + if (backend == ImageTransfer.Backend.file) { + final String filePath = cmd.getFile(); + if (StringUtils.isBlank(filePath)) { + return new CreateImageTransferAnswer(cmd, false, "file path is empty for file backend."); + } + payload.put("file", filePath); + } else { + String socket = cmd.getSocket(); + final String exportName = cmd.getExportName(); + if (StringUtils.isBlank(socket)) { + return new CreateImageTransferAnswer(cmd, false, "Empty socket."); + } + if (StringUtils.isBlank(exportName)) { + return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); + } + payload.put("socket", "/tmp/imagetransfer/" + socket + ".sock"); + payload.put("export", exportName); + String checkpointId = cmd.getCheckpointId(); + if (checkpointId != null) { + payload.put("export_bitmap", cmd.getCheckpointId()); + } + } + + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; + String listenAddress = resource.getImageServerListenAddress(); + if (StringUtils.isBlank(listenAddress)) { + listenAddress = resource.getPrivateIp(); + } + if (!startImageServerIfNotRunning(imageServerPort, listenAddress, resource)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to start image server."); + } + + if (!ImageServerControlSocket.registerTransfer(transferId, payload)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to register transfer with image server."); + } + + final String transferScheme = resource.isImageServerTlsEnabled() ? "https" : "http"; + final String transferUrl = String.format("%s://%s:%d/images/%s", transferScheme, listenAddress, imageServerPort, transferId); + return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java new file mode 100644 index 000000000000..ddb84ab29cb3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java @@ -0,0 +1,79 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.Map; + +import org.apache.cloudstack.backup.DeleteVmCheckpointCommand; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = DeleteVmCheckpointCommand.class) +public class LibvirtDeleteVmCheckpointCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(DeleteVmCheckpointCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return deleteBitmapsOnDisks(cmd); + } + return deleteDomainCheckpoint(cmd); + } + + private Answer deleteDomainCheckpoint(DeleteVmCheckpointCommand cmd) { + String vmName = cmd.getVmName(); + String checkpointId = cmd.getCheckpointId(); + String virshCmd = String.format("virsh checkpoint-delete %s %s", vmName, checkpointId); + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(virshCmd); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, "Failed to delete checkpoint: " + result); + } + return new Answer(cmd, true, "Checkpoint deleted"); + } + + /** + * Stopped VM: persistent bitmaps on disk images ({@code qemu-img bitmap --remove}), matching {@link LibvirtStartBackupCommandWrapper} bitmap --add. + */ + private Answer deleteBitmapsOnDisks(DeleteVmCheckpointCommand cmd) { + String checkpointId = cmd.getCheckpointId(); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + if (diskPathUuidMap == null || diskPathUuidMap.isEmpty()) { + return new Answer(cmd, false, "No disks provided for bitmap removal"); + } + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("qemu-img"); + script.add("bitmap"); + script.add("--remove"); + script.add(diskPath); + script.add(checkpointId); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, + "Failed to remove bitmap " + checkpointId + " from disk " + diskPath + ": " + result); + } + } + return new Answer(cmd, true, "Checkpoint bitmap removed from disks"); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java new file mode 100644 index 000000000000..3d9f6563d5eb --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -0,0 +1,101 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.FinalizeImageTransferCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = FinalizeImageTransferCommand.class) +public class LibvirtFinalizeImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private boolean stopImageServer(int imageServerPort, LibvirtComputingResource resource) { + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + logger.info("Image server not running, resetting failed state"); + resetService(unitName); + removeFirewallRule(imageServerPort); + return true; + } + + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + logger.info("Image server {} stopped", unitName); + + removeFirewallRule(imageServerPort); + + return true; + } + + private void removeFirewallRule(int port) { + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", port); + Script removeScript = new Script("/bin/bash", logger); + removeScript.add("-c"); + removeScript.add(String.format("iptables -D INPUT %s || true", rule)); + String result = removeScript.execute(); + if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { + logger.debug("Firewall rule removal result for port {}: {}", port, result); + } else { + logger.info("Firewall rule removed for port {} (or did not exist)", port); + } + } + + public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; + if (StringUtils.isBlank(transferId)) { + return new Answer(cmd, false, "transferId is empty."); + } + + int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); + if (activeTransfers < 0) { + logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); + stopImageServer(imageServerPort, resource); + return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); + } + + if (activeTransfers == 0) { + stopImageServer(imageServerPort, resource); + } + + return new Answer(cmd, true, "Image transfer finalized."); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java index 1536984f2e85..b1f64c8c6dbb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java @@ -49,10 +49,11 @@ public Answer execute(ScaleVmCommand command, LibvirtComputingResource libvirtCo conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); Domain dm = conn.domainLookupByName(vmName); - logger.debug(String.format("Scaling %s.", scalingDetails)); + logger.debug("Scaling {}.", scalingDetails); scaleMemory(dm, newMemory, vmDefinition); scaleVcpus(dm, newVcpus, vmDefinition); updateCpuShares(dm, newCpuShares); + libvirtComputingResource.updateCpuQuotaAndPeriod(dm, vmSpec, command.getLimitCpuUseChange()); return new ScaleVmAnswer(command, true, String.format("Successfully scaled %s.", scalingDetails)); } catch (LibvirtException | CloudRuntimeException e) { @@ -74,7 +75,7 @@ protected void updateCpuShares(Domain dm, int newCpuShares) throws LibvirtExcept if (oldCpuShares < newCpuShares) { LibvirtComputingResource.setCpuShares(dm, newCpuShares); - logger.info(String.format("Successfully increased cpu_shares of VM [%s] from [%s] to [%s].", dm.getName(), oldCpuShares, newCpuShares)); + logger.info("Successfully increased cpu_shares of VM [{}] from [{}] to [{}].", dm.getName(), oldCpuShares, newCpuShares); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java new file mode 100644 index 000000000000..629fd179afb2 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -0,0 +1,300 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.StartBackupAnswer; +import org.apache.cloudstack.backup.StartBackupCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.json.JSONArray; +import org.json.JSONObject; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartBackupCommand.class) +public class LibvirtStartBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return handleStoppedVmBackup(cmd, cmd.getToCheckpointId()); + } + return handleRunningVmBackup(cmd, resource); + } + + public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + String toCheckpointId = cmd.getToCheckpointId(); + String fromCheckpointId = cmd.getFromCheckpointId(); + Long fromCheckpointCreateTime = cmd.getFromCheckpointCreateTime(); + String socket = cmd.getSocket(); + + try { + if (StringUtils.isNotBlank(fromCheckpointId)) { + Answer redefineAnswer = ensureFromCheckpointExists(cmd, fromCheckpointId, fromCheckpointCreateTime); + if (redefineAnswer != null) { + return redefineAnswer; + } + } + + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + + // Create backup XML + String backupXml = createBackupXml(cmd, fromCheckpointId, socket, resource); + String checkpointXml = createCheckpointXml(toCheckpointId); + + // Write XMLs to temp files + File backupXmlFile = File.createTempFile("backup-", ".xml"); + File checkpointXmlFile = File.createTempFile("checkpoint-", ".xml"); + + try (FileWriter writer = new FileWriter(backupXmlFile)) { + writer.write(backupXml); + } + try (FileWriter writer = new FileWriter(checkpointXmlFile)) { + writer.write(checkpointXml); + } + + // Execute virsh backup-begin + String backupCmd = String.format("virsh backup-begin %s %s --checkpointxml %s", + vmName, backupXmlFile.getAbsolutePath(), checkpointXmlFile.getAbsolutePath()); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(backupCmd); + String result = script.execute(); + + backupXmlFile.delete(); + checkpointXmlFile.delete(); + + if (result != null) { + return new StartBackupAnswer(cmd, false, "Backup begin failed: " + result); + } + + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Backup started successfully", checkpointCreateTime); + + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Error starting backup: " + e.getMessage()); + } + } + + private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromCheckpointId, Long fromCheckpointCreateTime) { + String vmName = cmd.getVmName(); + Script dumpScript = new Script("/bin/bash"); + dumpScript.add("-c"); + dumpScript.add(String.format("virsh checkpoint-dumpxml --domain %s --checkpointname %s --no-domain", + vmName, fromCheckpointId)); + if (dumpScript.execute() == null) { + return null; + } + if (fromCheckpointCreateTime == null) { + return new StartBackupAnswer(cmd, false, "From checkpoint create time is null for checkpoint " + fromCheckpointId); + } + + String redefineXml = createCheckpointXmlForRedefine(fromCheckpointId, fromCheckpointCreateTime); + File redefineFile; + try { + redefineFile = File.createTempFile("checkpoint-redefine-", ".xml"); + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Failed to create temp file for checkpoint redefine: " + e.getMessage()); + } + try (FileWriter writer = new FileWriter(redefineFile)) { + writer.write(redefineXml); + } catch (Exception e) { + redefineFile.delete(); + return new StartBackupAnswer(cmd, false, "Failed to write checkpoint redefine XML: " + e.getMessage()); + } + String createCmd = String.format(LibvirtComputingResource.CHECKPOINT_CREATE_COMMAND, vmName, redefineFile.getAbsolutePath()); + Script createScript = new Script("/bin/bash"); + createScript.add("-c"); + createScript.add(createCmd); + String result = createScript.execute(); + redefineFile.delete(); + if (result != null) { + return new StartBackupAnswer(cmd, false, "Failed to redefine from-checkpoint " + fromCheckpointId + ": " + result); + } + return null; + } + + private String createCheckpointXmlForRedefine(String checkpointName, Long createTime) { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + xml.append(" ").append(checkpointName).append("\n"); + xml.append(" ").append(createTime).append("\n"); + xml.append(""); + return xml.toString(); + } + + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, String socket, LibvirtComputingResource resource) throws LibvirtException { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + + xml.append(String.format(" \n", socket)); + + xml.append(" \n"); + + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathLabelMap = resource.getDiskPathLabelMap(cmd.getVmName()); + Map diskPathHasFromCheckpointMap = new HashMap<>(); + if (StringUtils.isNotBlank(fromCheckpointId)) { + Domain vm = null; + try { + vm = resource.getDomain(resource.getLibvirtUtilitiesHelper().getConnection(), cmd.getVmName()); + if (vm != null) { + diskPathHasFromCheckpointMap = getVmDiskPathHasFromCheckpointMap(vm, fromCheckpointId); + } else { + logger.warn("Failed to get domain for VM [{}] while evaluating export bitmap [{}]. Falling back to full Backup", + cmd.getVmName(), fromCheckpointId); + } + } finally { + if (vm != null) { + vm.free(); + } + } + } + + for (Map.Entry entry : diskPathLabelMap.entrySet()) { + String diskPath = entry.getKey(); + if (!diskPathUuidMap.containsKey(diskPath)) { + continue; + } + String diskName = entry.getValue(); + String export = diskPathUuidMap.get(diskPath); + String scratchFile = "/var/tmp/scratch-" + export + ".qcow2"; + xml.append(" \n"); + xml.append(" \n"); + xml.append(" \n"); + } + + xml.append(" \n"); + xml.append(""); + + return xml.toString(); + } + + private String createCheckpointXml(String checkpointId) { + return "\n" + + " " + checkpointId + "\n" + + ""; + } + + private Answer handleStoppedVmBackup(StartBackupCommand cmd, String toCheckpointId) { + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathPassphraseMap = cmd.getDiskPathPassphraseMap(); + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("qemu-img"); + byte[] passphrase = diskPathPassphraseMap.get(diskPath); + + script.add("bitmap"); + script.add("--add"); + + if (passphrase != null && passphrase.length > 0) { + KeyFile srcKey; + try { + srcKey = new KeyFile(passphrase); + } catch (IOException ex) { + return new StartBackupAnswer(cmd, false, "Failed to create KeyFile while adding bitmap " + toCheckpointId + " to disk " + diskPath); + } + script.add("--object"); + script.add(String.format("secret,id=sec0,file=%s", srcKey)); + script.add("--image-opts"); + script.add(String.format("driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", diskPath)); + } else { + script.add(diskPath); + } + + script.add(toCheckpointId); + String result = script.execute(); + if (result != null) { + return new StartBackupAnswer(cmd, false, + "Failed to add bitmap " + toCheckpointId + " to disk " + diskPath + ": " + result); + } + } + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Stopped VM backup: checkpoint bitmap added successfully", + checkpointCreateTime); + } + + private long getCheckpointCreateTime() { + return System.currentTimeMillis() / 1000; + } + + private Map getVmDiskPathHasFromCheckpointMap(Domain vm, String fromCheckpointId) throws LibvirtException { + Map diskPathHasFromCheckpointMap = new HashMap<>(); + String queryBlock = vm.qemuMonitorCommand(QemuCommand.buildQemuCommand("query-block", null), 0); + JSONObject response = new JSONObject(queryBlock); + JSONArray blocks = response.optJSONArray("return"); + if (blocks == null) { + logger.warn("Couldn't get bitmap information for the VM [{}]. Falling back to full Backup", vm.getName()); + return diskPathHasFromCheckpointMap; + } + for (int i = 0; i < blocks.length(); i++) { + JSONObject block = blocks.getJSONObject(i); + JSONObject inserted = block.optJSONObject("inserted"); + if (inserted == null) { + continue; + } + String file = inserted.optString("file"); + if (StringUtils.isBlank(file)) { + continue; + } + JSONArray dirtyBitmaps = inserted.optJSONArray("dirty-bitmaps"); + boolean hasFromCheckpointBitmap = false; + if (dirtyBitmaps != null) { + for (int j = 0; j < dirtyBitmaps.length(); j++) { + JSONObject dirtyBitmap = dirtyBitmaps.optJSONObject(j); + if (dirtyBitmap == null) { + continue; + } + String bitmapName = dirtyBitmap.optString("name"); + if (fromCheckpointId.equals(bitmapName)) { + hasFromCheckpointBitmap = true; + break; + } + } + } + diskPathHasFromCheckpointMap.put(file, hasFromCheckpointBitmap); + } + return diskPathHasFromCheckpointMap; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java new file mode 100644 index 000000000000..661344677640 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -0,0 +1,197 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.IOException; + +import org.apache.cloudstack.backup.StartNBDServerAnswer; +import org.apache.cloudstack.backup.StartNBDServerCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.json.JSONArray; +import org.json.JSONObject; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartNBDServerCommand.class) +public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private String createSystemdRunCmd(StartNBDServerCommand cmd, String unitName, String volumePath, String exportName, String socket) throws IOException { + String socketName = "/tmp/imagetransfer/" + socket + ".sock"; + + String bitmapArg = ""; + if (StringUtils.isNotBlank(cmd.getFromCheckpointId()) + && isBitmapPresentOnDisk(volumePath, cmd.getFromCheckpointId())) { + bitmapArg = "-B " + cmd.getFromCheckpointId(); + } + + byte[] passphrase = cmd.getPassphrase(); // may be null + String readOnlyArg = cmd.getDirection().equals("download") ? "--read-only" : ""; + String imageArg = volumePath; + String secretArg = ""; + if (passphrase != null && passphrase.length > 0) { + KeyFile srcKey = new KeyFile(passphrase); + secretArg = String.format("--object secret,id=sec0,file=%s", srcKey); + imageArg = String.format("--image-opts driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", volumePath); + } + + // --persistent: Don't stop the service when the last client disconnects. + // --shared=NUM: Allow up to NUM clients to share the device (default 1), 0 for unlimited. Number of parallel connections is managed by the image server. + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no qemu-nbd %s " + + "--export-name %s --socket %s --persistent --shared=0 %s %s %s", + unitName, + secretArg, + exportName, + socketName, + bitmapArg, + readOnlyArg, + imageArg + ); + return systemdRunCmd; + } + + @Override + public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { + String volumePath = cmd.getVolumePath(); + String socket = cmd.getSocket(); + String exportName = cmd.getExportName(); + String transferId = cmd.getTransferId(); + + if (StringUtils.isBlank(volumePath)) { + return new StartNBDServerAnswer(cmd, false, "Volume path is required for the nbd server"); + } + if (StringUtils.isBlank(exportName)) { + return new StartNBDServerAnswer(cmd, false, "Export name is required for the nbd server"); + } + if (StringUtils.isBlank(socket)) { + return new StartNBDServerAnswer(cmd, false, "Socket is required for the nbd server"); + } + + String unitName = "qemu-nbd-" + transferId.hashCode(); + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null) { + return new StartNBDServerAnswer(cmd, false, "A qemu-nbd service is already running on the port."); + } + + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + + String systemdRunCmd = ""; + try { + systemdRunCmd = createSystemdRunCmd(cmd, unitName, volumePath, exportName, socket); + } catch (IOException e) { + logger.error("Failed to create the KeyFile for qemu-nbd service", e); + return new StartNBDServerAnswer(cmd, false, "Failed to create systemd run command for qemu-nbd service: " + e.getMessage()); + } + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start qemu-nbd service: %s", startResult)); + return new StartNBDServerAnswer(cmd, false, "Failed to start qemu-nbd service: " + startResult); + } + + // Wait with timeout until the service is up + int maxWaitSeconds = 20; + int pollIntervalMs = 5000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serviceActive = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new StartNBDServerAnswer(cmd, false, "Interrupted while waiting for qemu-nbd service to start"); + } + Script verifyScript = new Script("/bin/bash", logger); + verifyScript.add("-c"); + verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String verifyResult = verifyScript.execute(); + if (verifyResult == null) { + serviceActive = true; + logger.info(String.format("qemu-nbd service %s is now active (attempt %d)", unitName, attempt + 1)); + break; + } + } + + if (!serviceActive) { + logger.error(String.format("qemu-nbd service %s failed to become active within %d seconds", unitName, maxWaitSeconds)); + return new StartNBDServerAnswer(cmd, false, + String.format("qemu-nbd service failed to start within %d seconds", maxWaitSeconds)); + } + + String transferUrl = String.format("nbd+unix:///%s", cmd.getSocket()); + return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for upload", + transferId, transferUrl); + } + + private boolean isBitmapPresentOnDisk(String volumePath, String fromCheckpointId) { + String qemuImgInfo = Script.runBashScriptIgnoreExitValue( + String.format("qemu-img info --output=json %s", volumePath), 0); + if (StringUtils.isBlank(qemuImgInfo)) { + logger.warn("Unable to read qemu-img info output for disk path [{}].", volumePath); + return false; + } + try { + JSONObject info = new JSONObject(qemuImgInfo); + JSONObject formatSpecific = info.optJSONObject("format-specific"); + if (formatSpecific == null) { + return false; + } + JSONObject formatData = formatSpecific.optJSONObject("data"); + if (formatData == null) { + return false; + } + JSONArray bitmaps = formatData.optJSONArray("bitmaps"); + if (bitmaps == null) { + return false; + } + for (int i = 0; i < bitmaps.length(); i++) { + JSONObject bitmap = bitmaps.optJSONObject(i); + if (bitmap == null) { + continue; + } + if (fromCheckpointId.equals(bitmap.optString("name"))) { + return true; + } + } + } catch (Exception e) { + logger.warn("Failed to parse qemu-img info output for disk path [{}].", volumePath, e); + } + return false; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java new file mode 100644 index 000000000000..1185d89bc0b3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopBackupAnswer; +import org.apache.cloudstack.backup.StopBackupCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.libvirt.Connect; +import org.libvirt.Domain; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopBackupCommand.class) +public class LibvirtStopBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StopBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + + try { + Connect conn = LibvirtConnection.getConnection(); + Domain dm = conn.domainLookupByName(vmName); + + if (dm == null) { + return new StopBackupAnswer(cmd, false, "Domain not found: " + vmName); + } + + // Execute virsh domjobabort + String abortCmd = String.format("virsh domjobabort %s", vmName); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(abortCmd); + String result = script.execute(); + + if (result != null && !result.isEmpty()) { + // Job abort may fail if no job is running, which is acceptable + logger.debug("domjobabort result: " + result); + } + + return new StopBackupAnswer(cmd, true, "Backup stopped successfully"); + + } catch (Exception e) { + return new StopBackupAnswer(cmd, false, "Error stopping backup: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java new file mode 100644 index 000000000000..57c7ebb706bc --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java @@ -0,0 +1,72 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopNBDServerCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopNBDServerCommand.class) +public class LibvirtStopNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + @Override + public Answer execute(StopNBDServerCommand cmd, LibvirtComputingResource resource) { + try { + String unitName = "qemu-nbd-" + cmd.getTransferId().hashCode(); + + // Check if the service is running + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + // Service is not running, but still reset-failed to clear any stale state + logger.info(String.format("qemu-nbd service %s is not running, resetting failed state", unitName)); + resetService(unitName); + return new Answer(cmd, true, "Image transfer finalized"); + } + + // Stop the systemd service + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + + return new Answer(cmd, true, "Image transfer finalized"); + + } catch (Exception e) { + logger.error("Error finalizing image transfer for upload", e); + return new Answer(cmd, false, "Error finalizing image transfer: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java index f5bfd898a4f4..8cf6de68f955 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java @@ -208,12 +208,12 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 8dd2116e1235..3e35ed9476b1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -33,35 +33,33 @@ public interface KVMStoragePool { - public static final long HeartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); - public static final long HeartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); - public static final long HeartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); - public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); - public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); + long HeartBeatUpdateTimeoutInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); + long HeartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + long HeartBeatCheckerTimeoutInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); - public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { + default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase); } - public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); + KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); - public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); + KVMPhysicalDisk createPhysicalDisk(String volumeUuid, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); - public boolean connectPhysicalDisk(String volumeUuid, Map details); + boolean connectPhysicalDisk(String volumeUuid, Map details); - public KVMPhysicalDisk getPhysicalDisk(String volumeUuid); + KVMPhysicalDisk getPhysicalDisk(String volumeUuid); - public boolean disconnectPhysicalDisk(String volumeUuid); + boolean disconnectPhysicalDisk(String volumeUuid); - public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format); + boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format); - public List listPhysicalDisks(); + List listPhysicalDisks(); - public String getUuid(); + String getUuid(); - public long getCapacity(); + long getCapacity(); - public long getUsed(); + long getUsed(); default Long getCapacityIops() { return null; @@ -71,51 +69,51 @@ default Long getUsedIops() { return null; } - public long getAvailable(); + long getAvailable(); - public boolean refresh(); + boolean refresh(); - public boolean isExternalSnapshot(); + boolean isExternalSnapshot(); - public String getLocalPath(); + String getLocalPath(); - public String getSourceHost(); + String getSourceHost(); - public String getSourceDir(); + String getSourceDir(); - public int getSourcePort(); + int getSourcePort(); - public String getAuthUserName(); + String getAuthUserName(); - public String getAuthSecret(); + String getAuthSecret(); - public StoragePoolType getType(); + StoragePoolType getType(); - public boolean delete(); + boolean delete(); PhysicalDiskFormat getDefaultFormat(); - public boolean createFolder(String path); + boolean createFolder(String path); - public boolean supportsConfigDriveIso(); + boolean supportsConfigDriveIso(); - public Map getDetails(); + Map getDetails(); default String getLocalPathFor(String relativePath) { return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); } - public boolean isPoolSupportHA(); + boolean isPoolSupportHA(); - public String getHearthBeatPath(); + String getHearthBeatPath(); - public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); + String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); - public String getStorageNodeId(); + String getStorageNodeId(); - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host); + Boolean hasHeartBeat(HAStoragePool pool, HostTO host); - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration); + Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration); default LibvirtVMDef.DiskDef.BlockIOSize getSupportedLogicalBlockSize() { return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 6e03b84d20cf..f95ebff5326f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -2047,7 +2047,7 @@ private void waitForBackup(String vmName) throws CloudRuntimeException { try { Thread.sleep(10000); } catch (InterruptedException e) { - throw new CloudRuntimeException(e); + logger.trace("Thread that was tracking the progress for backup of VM [{}] was interrupted. Ignoring.", vmName); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index 45c22d3ac754..910f0eb15e0b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -345,16 +345,14 @@ public String getHearthBeatPath() { public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { - Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeout, logger); + Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); cmd.add("-i", primaryStoragePool.getPoolIp()); cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); cmd.add("-m", primaryStoragePool.getMountDestPath()); if (hostValidation) { cmd.add("-h", hostPrivateIp); - } - - if (!hostValidation) { + } else { cmd.add("-c"); } @@ -372,53 +370,53 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { - boolean validResult = false; + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { String hostIp = host.getPrivateNetwork().getIp(); - Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeout, logger); + Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); cmd.add("-i", pool.getPoolIp()); cmd.add("-p", pool.getPoolMountSourcePath()); cmd.add("-m", pool.getMountDestPath()); cmd.add("-h", hostIp); cmd.add("-r"); - cmd.add("-t", String.valueOf(HeartBeatUpdateFreq / 1000)); + cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); - logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, - pool.getPoolIp())); + logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned [%s]. [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), - result, parsedLine, hostIp)); + logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + return false; } else { - validResult = true; + logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString()); + return true; } - return validResult; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + String hostIp = host.getPrivateNetwork().getIp(); Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); cmd.add("-i", pool.getPoolIp()); cmd.add("-p", pool.getPoolMountSourcePath()); cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", host.getPrivateNetwork().getIp()); + cmd.add("-h", hostIp); cmd.add("-u", volumeUUIDListString); - cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000))); + cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); cmd.add("-d", String.valueOf(duration)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); - logger.debug(String.format("Checking heart beat with KVMHAVMActivityChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, pool.getPoolIp())); + logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn(String.format("Checking heart beat with KVMHAVMActivityChecker command [%s] returned [%s]. It is [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), result, parsedLine, host.getPrivateNetwork().getIp())); + logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); return false; } else { + logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString()); return true; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java index 229481b1f795..df9986c99194 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java @@ -225,13 +225,13 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, - String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java index e8243c3f7cfe..fc512ff94a94 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java @@ -236,12 +236,12 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java index 59ea720328f5..3fbb5340fcce 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java @@ -19,38 +19,37 @@ import org.apache.cloudstack.framework.config.ConfigKey; -public class KVMHAConfig { +public interface KVMHAConfig { - public static final ConfigKey KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10", + ConfigKey KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10", "The maximum length of time, in seconds, expected for an health check to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60", + ConfigKey KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60", "The maximum length of time, in seconds, expected for an activity check to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60", + ConfigKey KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60", "The interval, in seconds, between activity checks.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10", + ConfigKey KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10", "The maximum number of activity check attempts to perform before deciding to recover or degrade a resource.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7", + ConfigKey KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7", "The activity check failure threshold ratio. This is used with the activity check maximum attempts for deciding to recover or degrade a resource. For most environments, please keep this value above 0.5.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300", + ConfigKey KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300", "The maximum length of time, in seconds, a resource can be in degraded state where only health checks are performed.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60", + ConfigKey KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60", "The maximum length of time, in seconds, expected for a recovery operation to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600", + ConfigKey KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600", "The maximum length of time, in seconds, to wait for a resource to recover.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1", + ConfigKey KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1", "The maximum recovery attempts to be made for a resource, after which the resource is fenced. The recovery counter resets when a health check passes for a resource.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", + ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster); - } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index b937be5265b7..f0b5cfc337de 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -68,17 +68,18 @@ public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACh @Override public boolean recover(Host r) throws HARecoveryException { + logger.debug("Recover the host {}", r); try { - if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ + if (outOfBandManagementService.isOutOfBandManagementEnabled(r)) { final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); return resp.getSuccess(); } else { logger.warn("OOBM recover operation failed for the host {}", r); return false; } - } catch (Exception e){ + } catch (Exception e) { logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage()); - throw new HARecoveryException(String.format(" OOBM service is not configured or enabled for this host %s", r), e); + throw new HARecoveryException(String.format("OOBM service is not configured or enabled for this host %s", r), e); } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java index 31f87d7e0442..af7441c4fd29 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java @@ -19,6 +19,7 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckOnHostAnswer; import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.dc.dao.ClusterDao; @@ -61,7 +62,7 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Inject private AgentManager agentMgr; @Inject - private PrimaryDataStoreDao storagePool; + private PrimaryDataStoreDao storagePoolDao; @Inject private StorageManager storageManager; @Inject @@ -70,11 +71,11 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Override public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException { try { - return isVMActivityOnHost(r, suspectTime); + return hasVMActivityOnHost(r, suspectTime); } catch (HACheckerException e) { - //Re-throwing the exception to avoid poluting the 'HACheckerException' already thrown + //Re-throwing the exception to avoid polluting the 'HACheckerException' already thrown throw e; - } catch (Exception e){ + } catch (Exception e) { String message = String.format("Operation timed out, probably the %s is not reachable.", r.toString()); logger.warn(message, e); throw new HACheckerException(message, e); @@ -83,82 +84,115 @@ public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException @Override public boolean isHealthy(Host r) { - return isAgentActive(r); + return isHostAgentUp(r); } - private boolean isAgentActive(Host agent) { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { - throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType())); + private boolean isHostAgentUp(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", host.getHypervisorType())); + } + + Status hostStatus = getHostAgentStatus(host); + + logger.debug("{} has the status [{}].", host.toString(), hostStatus); + return hostStatus == Status.Up; + } + + public Status getHostAgentStatus(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + return null; + } + + Status hostStatusFromItself = checkHostStatusWithSameHost(host); + if (hostStatusFromItself == Status.Up) { + return Status.Up; } - Status hostStatus = Status.Unknown; - Status neighbourStatus = Status.Unknown; - final CheckOnHostCommand cmd = new CheckOnHostCommand(agent, HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value()); + + Status hostStatusFromNeighbour = checkHostStatusWithNeighbourHosts(host); + Status hostStatus = hostStatusFromItself; + if (hostStatusFromNeighbour == Status.Up && (hostStatusFromItself == Status.Disconnected || hostStatusFromItself == Status.Down)) { + hostStatus = Status.Disconnected; + } + if (hostStatusFromNeighbour == Status.Down && (hostStatusFromItself == Status.Disconnected || hostStatusFromItself == Status.Down)) { + hostStatus = Status.Down; + } + + logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, host); + return hostStatus; + } + + private Status checkHostStatusWithSameHost(Host host) { + Status hostStatus; + boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); + final CheckOnHostCommand cmd = new CheckOnHostCommand(host, reportFailureIfOneStorageIsDown); try { - logger.debug(String.format("Checking %s status...", agent.toString())); - Answer answer = agentMgr.easySend(agent.getId(), cmd); + logger.debug("Checking {} status...", host.toString()); + Answer answer = agentMgr.easySend(host.getId(), cmd); if (answer != null) { - hostStatus = answer.getResult() ? Status.Down : Status.Up; - logger.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus)); - - if ( hostStatus == Status.Up ){ - return true; + if (answer.getResult()) { + hostStatus = ((CheckOnHostAnswer)answer).isAlive() ? Status.Up : Status.Down; + } else { + logger.debug("{} is not active according to itself, details: {}.", host.toString(), answer.getDetails()); + hostStatus = Status.Down; } - } - else { - logger.debug(String.format("Setting %s to \"Disconnected\" status.", agent.toString())); + logger.debug("{} has the status [{}].", host.toString(), hostStatus); + } else { + logger.debug("Setting {} to \"Disconnected\" status.", host.toString()); hostStatus = Status.Disconnected; } } catch (Exception e) { - logger.warn(String.format("Failed to send command CheckOnHostCommand to %s.", agent.toString()), e); + logger.warn("Failed to send command CheckOnHostCommand to {}.", host.toString(), e); + hostStatus = Status.Disconnected; } - List neighbors = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); + return hostStatus; + } + + private Status checkHostStatusWithNeighbourHosts(Host host) { + Status hostStatusFromNeighbour = Status.Unknown; + boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); + final CheckOnHostCommand cmd = new CheckOnHostCommand(host, reportFailureIfOneStorageIsDown); + List neighbors = resourceManager.listHostsInClusterByStatus(host.getClusterId(), Status.Up); for (HostVO neighbor : neighbors) { - if (neighbor.getId() == agent.getId() || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { + if (neighbor.getId() == host.getId() + || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { continue; } try { - logger.debug(String.format("Investigating %s via neighbouring %s.", agent.toString(), neighbor.toString())); - + logger.debug("Investigating {} via neighboring {}.", host.toString(), neighbor.toString()); Answer answer = agentMgr.easySend(neighbor.getId(), cmd); if (answer != null) { - neighbourStatus = answer.getResult() ? Status.Down : Status.Up; - - logger.debug(String.format("Neighbouring %s returned status [%s] for the investigated %s.", neighbor.toString(), neighbourStatus, agent.toString())); - - if (neighbourStatus == Status.Up) { - break; + if (answer.getResult()) { + hostStatusFromNeighbour = ((CheckOnHostAnswer)answer).isAlive() ? Status.Up : Status.Down; + logger.debug("Neighboring {} returned status [{}] for the investigated {}.", neighbor.toString(), hostStatusFromNeighbour, host.toString()); + if (hostStatusFromNeighbour == Status.Up) { + return hostStatusFromNeighbour; + } + } else { + logger.debug("{} is not active according to neighbor {}, details: {}.", host.toString(), neighbor.toString(), answer.getDetails()); } } else { - logger.debug(String.format("Neighbouring %s is Disconnected.", neighbor.toString())); + logger.debug("Neighboring {} is Disconnected.", neighbor.toString()); } } catch (Exception e) { - logger.warn(String.format("Failed to send command CheckOnHostCommand to %s.", neighbor.toString()), e); + logger.warn("Failed to send command CheckOnHostCommand to neighbor {}.", neighbor.toString(), e); } } - if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Disconnected; - } - if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Down; - } - logger.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus)); - - return hostStatus == Status.Up; + return hostStatusFromNeighbour; } - private boolean isVMActivityOnHost(Host agent, DateTime suspectTime) throws HACheckerException { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { - throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType())); + private boolean hasVMActivityOnHost(Host host, DateTime suspectTime) throws HACheckerException { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", host.getHypervisorType())); } boolean activityStatus = true; - HashMap> poolVolMap = getVolumeUuidOnHost(agent); - for (StoragePool pool : poolVolMap.keySet()) { - activityStatus = verifyActivityOfStorageOnHost(poolVolMap, pool, agent, suspectTime, activityStatus); + HashMap> poolVolumeMap = getStoragePoolAndVolumeInfoOnHost(host); + for (StoragePool pool : poolVolumeMap.keySet()) { + activityStatus = verifyActivityOfStorageOnHost(poolVolumeMap, pool, host, suspectTime, activityStatus); if (!activityStatus) { - logger.warn("It seems that the storage pool [{}] does not have activity on {}.", pool, agent); + logger.warn("It seems that the storage pool [{}] does not have activity on {}.", pool, host); break; } } @@ -166,66 +200,64 @@ private boolean isVMActivityOnHost(Host agent, DateTime suspectTime) throws HACh return activityStatus; } - protected boolean verifyActivityOfStorageOnHost(HashMap> poolVolMap, StoragePool pool, Host agent, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException { + protected boolean verifyActivityOfStorageOnHost(HashMap> poolVolMap, StoragePool pool, Host host, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException { List volume_list = poolVolMap.get(pool); - final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime); + final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(host, pool, volume_list, suspectTime); - logger.debug("Checking VM activity for {} on storage pool [{}].", agent.toString(), pool); + logger.debug("Checking VM activity for {} on storage pool [{}].", host.toString(), pool); try { - Answer answer = storageManager.sendToPool(pool, getNeighbors(agent), cmd); - + Answer answer = storageManager.sendToPool(pool, getNeighbors(host), cmd); if (answer != null) { activityStatus = !answer.getResult(); - logger.debug("{} {} activity on storage pool [{}]", agent.toString(), activityStatus ? "has" : "does not have", pool); + logger.debug("{} {} activity on storage pool [{}]", host.toString(), activityStatus ? "has" : "does not have", pool); } else { - String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", agent.toString(), pool); + String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", host.toString(), pool); logger.debug(message); throw new IllegalStateException(message); } - } catch (StorageUnavailableException e){ - String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool, agent); + } catch (StorageUnavailableException e) { + String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool, host); logger.warn(message, e); throw new HACheckerException(message, e); } return activityStatus; } - private HashMap> getVolumeUuidOnHost(Host agent) { - List vm_list = vmInstanceDao.listByHostId(agent.getId()); - List volume_list = new ArrayList(); - for (VirtualMachine vm : vm_list) { + private HashMap> getStoragePoolAndVolumeInfoOnHost(Host host) { + List vmListOnHost = vmInstanceDao.listByHostId(host.getId()); + List volumeListOnHost = new ArrayList<>(); + for (VirtualMachine vm : vmListOnHost) { logger.debug("Retrieving volumes of VM [{}]...", vm); - List vm_volume_list = volumeDao.findByInstance(vm.getId()); - volume_list.addAll(vm_volume_list); + List volumeListOfVM = volumeDao.findByInstance(vm.getId()); + volumeListOnHost.addAll(volumeListOfVM); } - HashMap> poolVolMap = new HashMap>(); - for (Volume vol : volume_list) { - StoragePool sp = storagePool.findById(vol.getPoolId()); - logger.debug("Retrieving storage pool [{}] of volume [{}]...", sp, vol); - if (!poolVolMap.containsKey(sp)) { - List list = new ArrayList(); - list.add(vol); + HashMap> poolVolumeMap = new HashMap<>(); + for (Volume volume : volumeListOnHost) { + StoragePool pool = storagePoolDao.findById(volume.getPoolId()); + logger.debug("Retrieving storage pool [{}] of volume [{}]...", pool, volume); + if (!poolVolumeMap.containsKey(pool)) { + List volList = new ArrayList<>(); + volList.add(volume); - poolVolMap.put(sp, list); + poolVolumeMap.put(pool, volList); } else { - poolVolMap.get(sp).add(vol); + poolVolumeMap.get(pool).add(volume); } } - return poolVolMap; + return poolVolumeMap; } - public long[] getNeighbors(Host agent) { - List neighbors = new ArrayList(); - List cluster_hosts = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); - logger.debug("Retrieving all \"Up\" hosts from cluster [{}]...", clusterDao.findById(agent.getClusterId())); - for (HostVO host : cluster_hosts) { - if (host.getId() == agent.getId() || (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { + public long[] getNeighbors(Host host) { + List neighbors = new ArrayList<>(); + List clusterHosts = resourceManager.listHostsInClusterByStatus(host.getClusterId(), Status.Up); + logger.debug("Retrieving all \"Up\" hosts from cluster [{}]...", clusterDao.findById(host.getClusterId())); + for (HostVO clusterHost : clusterHosts) { + if (clusterHost.getId() == host.getId() || (clusterHost.getHypervisorType() != Hypervisor.HypervisorType.KVM && clusterHost.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { continue; } - neighbors.add(host.getId()); + neighbors.add(clusterHost.getId()); } return ArrayUtils.toPrimitive(neighbors.toArray(new Long[neighbors.size()])); } - } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index b96295240076..b3bdafb73751 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -3133,7 +3133,7 @@ public void testCheckOnHostCommand() { assertNotNull(wrapper); final Answer answer = wrapper.execute(command, libvirtComputingResourceMock); - assertTrue(answer.getResult()); + assertFalse(answer.getResult()); verify(libvirtComputingResourceMock, times(1)).getMonitor(); } @@ -5642,35 +5642,45 @@ public void testAddExtraConfigComponentNotEmptyExtraConfig() { Mockito.verify(vmDef, times(1)).addComp(any()); } - public void validateGetCurrentMemAccordingToMemBallooningWithoutMemBalooning(){ + @Test + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryWithoutMemBallooning(){ VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); - Mockito.when(vmTo.getType()).thenReturn(Type.User); LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); libvirtComputingResource.noMemBalloon = true; - long maxMemory = 2048; + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; - long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, maxMemory); - Assert.assertEquals(maxMemory, currentMemory); - Mockito.verify(vmTo, Mockito.times(0)).getMinRam(); + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(requestedMemory, currentMemory); } @Test - public void validateGetCurrentMemAccordingToMemBallooningWithtMemBalooning(){ + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryWithMemoryBallooning(){ LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); libvirtComputingResource.noMemBalloon = false; - long maxMemory = 2048; - long minMemory = ByteScaleUtils.mebibytesToBytes(64); - VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); Mockito.when(vmTo.getType()).thenReturn(Type.User); - Mockito.when(vmTo.getMinRam()).thenReturn(minMemory); + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; - long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, maxMemory); - Assert.assertEquals(ByteScaleUtils.bytesToKibibytes(minMemory), currentMemory); - Mockito.verify(vmTo).getMinRam(); + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(minMemory, currentMemory); } + @Test + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryForSystemVms() { + LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); + libvirtComputingResource.noMemBalloon = false; + + VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); + Mockito.when(vmTo.getType()).thenReturn(Type.SecondaryStorageVm); + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; + + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(requestedMemory, currentMemory); + } @Test public void validateCreateGuestResourceDefWithVcpuMaxLimit(){ LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); @@ -7234,4 +7244,82 @@ public void getInterfaceTestInvalidMacAddressThrowCloudRuntimeException() { libvirtComputingResourceSpy.getInterface(connMock, vmName, invalidMacAddress); } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreNotUpdatedWhenLibvirtVersionIsLessThanTheMinimum() throws LibvirtException { + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 8999; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, null, false); + Mockito.verify(domainMock, Mockito.never()).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreNotUpdatedWhenThereIsNoCapCapChangeAndNoCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, false); + Mockito.verify(domainMock, Mockito.never()).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertQuotaIsRemovedWhenThereIsCpuCapChangeAndNoCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, true); + Mockito.verify(domainMock, Mockito.times(1)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreUpdatedWhenThereIsNotCpuCapChangeAndCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); + double cpuQuotaPercentage = 0.03; + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(cpuQuotaPercentage); + Mockito.doReturn(new Pair<>(1000, 300L)).when(libvirtComputingResourceSpy).getPeriodAndQuota(cpuQuotaPercentage); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, false); + Mockito.verify(domainMock, Mockito.times(2)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreUpdatedWhenThereIsCpuCapChangeAndCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); + double cpuQuotaPercentage = 0.03; + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(cpuQuotaPercentage); + Mockito.doReturn(new Pair<>(1000, 300L)).when(libvirtComputingResourceSpy).getPeriodAndQuota(cpuQuotaPercentage); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, true); + Mockito.verify(domainMock, Mockito.times(2)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void getPeriodAndQuotaTestAssertQuotaIsEqualToPeriodMultipliedByQuotaPercentage() { + double cpuQuotaPercentage = 0.3; + int expectedPeriod = CpuTuneDef.DEFAULT_PERIOD; + long expectedQuota = (long) (expectedPeriod * cpuQuotaPercentage); + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void getPeriodAndQuotaTestQuotaIsEqualToMinimumWhenRequired() { + double cpuQuotaPercentage = 0.03; + long expectedQuota = CpuTuneDef.MIN_QUOTA; + int expectedPeriod = (int) ((double) expectedQuota / cpuQuotaPercentage); + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void getPeriodAndQuotaTestPeriodIsEqualToMaximumWhenRequired() { + double cpuQuotaPercentage = 0.0003; + long expectedQuota = CpuTuneDef.MIN_QUOTA; + int expectedPeriod = CpuTuneDef.MAX_PERIOD; + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java new file mode 100644 index 000000000000..8782f592aed3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCreateImageTransferCommandWrapperTest { + + private LibvirtCreateImageTransferCommandWrapper wrapper; + private CreateImageTransferCommand command; + private LibvirtComputingResource resource; + + @Before + public void setUp() { + wrapper = new LibvirtCreateImageTransferCommandWrapper(); + command = Mockito.mock(CreateImageTransferCommand.class); + resource = Mockito.mock(LibvirtComputingResource.class); + } + + @Test + public void testExecuteBlankTransferIdReturnsFailure() { + Mockito.when(command.getTransferId()).thenReturn(""); + Mockito.when(command.getBackend()).thenReturn(ImageTransfer.Backend.nbd); + + Answer answer = wrapper.execute(command, resource); + + Assert.assertFalse(answer.getResult()); + Assert.assertEquals("transferId is empty.", answer.getDetails()); + } + + @Test + public void testExecuteNbdBackendSuccessReturnsUrl() { + Mockito.when(command.getTransferId()).thenReturn("tr-1"); + Mockito.when(command.getBackend()).thenReturn(ImageTransfer.Backend.nbd); + Mockito.when(command.getIdleTimeoutSeconds()).thenReturn(120); + Mockito.when(command.getSocket()).thenReturn("sock-1"); + Mockito.when(command.getExportName()).thenReturn("vol-1"); + Mockito.when(command.getCheckpointId()).thenReturn("cp-1"); + + Mockito.when(resource.getImageServerPath()).thenReturn("/opt/cloudstack/image/server.py"); + Mockito.when(resource.getImageServerListenAddress()).thenReturn(""); + Mockito.when(resource.getPrivateIp()).thenReturn("10.0.0.10"); + Mockito.when(resource.isImageServerTlsEnabled()).thenReturn(false); + + try (MockedConstruction