diff --git a/.asf.yaml b/.asf.yaml index a19e3a60a2ba..3b2f7691d136 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -50,10 +50,13 @@ github: rebase: false collaborators: - - gpordeus + - ingox + - gp-santos - erikbocks - Imvedansh - Damans227 + - jmsperu + - GaOrtiga protected_branches: ~ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0602871ef1e..689275cff115 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /plugins/storage/volume/linstor @rp- /plugins/storage/volume/storpool @slavkap +/plugins/storage/volume/ontap @rajiv1 @sandeeplocharla @piyush5 @suryag .pre-commit-config.yaml @jbampton /.github/linters/ @jbampton diff --git a/.github/actions/install-nonoss/action.yml b/.github/actions/install-nonoss/action.yml new file mode 100644 index 000000000000..39a03213c29d --- /dev/null +++ b/.github/actions/install-nonoss/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Install CloudStack Non-OSS' +description: 'Clones and installs the shapeblue/cloudstack-nonoss repository.' + +runs: + using: "composite" + steps: + - name: Install cloudstack-nonoss + shell: bash + run: | + git clone --depth 1 https://github.com/shapeblue/cloudstack-nonoss.git nonoss + cd nonoss + bash -x install-non-oss.sh + cd .. + rm -fr nonoss diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 000000000000..0f8425229242 --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup CloudStack Environment' +description: 'Sets up JDK (with Maven cache), optionally Python, and optionally APT build dependencies for CloudStack.' + +inputs: + java-version: + description: 'The JDK version to use' + required: false + default: '17' + install-python: + description: 'Whether to install Python 3.10' + required: false + default: 'false' + install-apt-deps: + description: 'Whether to install CloudStack APT build dependencies' + required: false + default: 'false' + +runs: + using: "composite" + steps: + - name: Set up JDK ${{ inputs.java-version }} + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: ${{ inputs.java-version }} + distribution: 'adopt' + architecture: x64 + cache: 'maven' + + - name: Set up Python + if: ${{ inputs.install-python == 'true' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10' + architecture: x64 + + - name: Install Build Dependencies + if: ${{ inputs.install-apt-deps == 'true' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 000000000000..ea25ffab6b9e --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,56 @@ +{ + "entries": { + "actions/github-script@v9.0.0": { + "repo": "actions/github-script", + "version": "v9.0.0", + "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" + }, + "github/gh-aw-actions/setup@v0.76.1": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.76.1", + "sha": "46d564922b082d0db93244972e8005ea6904ee5f" + } + }, + "containers": { + "ghcr.io/github/gh-aw-firewall/agent:0.18.0": { + "image": "ghcr.io/github/gh-aw-firewall/agent:0.18.0", + "digest": "sha256:ab84dfc7f5998cb8cd0c596526dd573b7e7d06c6a740266a1e6df879fa16c866", + "pinned_image": "ghcr.io/github/gh-aw-firewall/agent:0.18.0@sha256:ab84dfc7f5998cb8cd0c596526dd573b7e7d06c6a740266a1e6df879fa16c866" + }, + "ghcr.io/github/gh-aw-firewall/agent:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/agent:0.25.55", + "digest": "sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731", + "pinned_image": "ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731" + }, + "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55", + "digest": "sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3", + "pinned_image": "ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3" + }, + "ghcr.io/github/gh-aw-firewall/squid:0.18.0": { + "image": "ghcr.io/github/gh-aw-firewall/squid:0.18.0", + "digest": "sha256:82a5d062a5612a57a43a171a5b79ddbb690a86a8ddda02339cc1675131ae9f8b", + "pinned_image": "ghcr.io/github/gh-aw-firewall/squid:0.18.0@sha256:82a5d062a5612a57a43a171a5b79ddbb690a86a8ddda02339cc1675131ae9f8b" + }, + "ghcr.io/github/gh-aw-firewall/squid:0.25.55": { + "image": "ghcr.io/github/gh-aw-firewall/squid:0.25.55", + "digest": "sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca", + "pinned_image": "ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca" + }, + "ghcr.io/github/gh-aw-mcpg:v0.1.4": { + "image": "ghcr.io/github/gh-aw-mcpg:v0.1.4", + "digest": "sha256:0acf25aa1d409f9c73be9e39ac84f4bd4b90d8bfa1db4dc6d7f47d38ccd58914", + "pinned_image": "ghcr.io/github/gh-aw-mcpg:v0.1.4@sha256:0acf25aa1d409f9c73be9e39ac84f4bd4b90d8bfa1db4dc6d7f47d38ccd58914" + }, + "ghcr.io/github/gh-aw-mcpg:v0.3.19": { + "image": "ghcr.io/github/gh-aw-mcpg:v0.3.19", + "digest": "sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f", + "pinned_image": "ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f" + }, + "ghcr.io/github/github-mcp-server:v0.30.3": { + "image": "ghcr.io/github/github-mcp-server:v0.30.3", + "digest": "sha256:a2b5fb79b1cee851bfc3532dfe480c3dc5736974ca9d93a7a9f68e52ce4b62a0", + "pinned_image": "ghcr.io/github/github-mcp-server:v0.30.3@sha256:a2b5fb79b1cee851bfc3532dfe480c3dc5736974ca9d93a7a9f68e52ce4b62a0" + } + } +} diff --git a/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md new file mode 100644 index 000000000000..72d61b434efa --- /dev/null +++ b/.github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md @@ -0,0 +1,73 @@ +--- +# Report formatting guidelines +--- + +## Report Structure Guidelines + +### 1. Header Levels +**Use h3 (###) or lower for all headers in your issue report to maintain proper document hierarchy.** + +When creating GitHub issues or discussions: +- Use `###` (h3) for main sections (e.g., "### Test Summary") +- Use `####` (h4) for subsections (e.g., "#### Device-Specific Results") +- Never use `##` (h2) or `#` (h1) in reports - these are reserved for titles + +### 2. Progressive Disclosure +**Wrap detailed test results in `
Section Name` tags to improve readability and reduce scrolling.** + +Use collapsible sections for: +- Verbose details (full test logs, raw data) +- Secondary information (minor warnings, extra context) +- Per-item breakdowns when there are many items + +Always keep critical information visible (summary, critical issues, key metrics). + +### 3. Report Structure Pattern + +1. **Overview**: 1-2 paragraphs summarizing key findings +2. **Critical Information**: Show immediately (summary stats, critical issues) +3. **Details**: Use `
Section Name` for expanded content +4. **Context**: Add helpful metadata (workflow run, date, trigger) + +### Design Principles (Airbnb-Inspired) + +Reports should: +- **Build trust through clarity**: Most important info immediately visible +- **Exceed expectations**: Add helpful context like trends, comparisons +- **Create delight**: Use progressive disclosure to reduce overwhelm +- **Maintain consistency**: Follow patterns across all reports + +### Example Report Structure + +```markdown +### Summary +- Key metric 1: value +- Key metric 2: value +- Status: ✅/⚠️/❌ + +### Critical Issues +[Always visible - these are important] + +
+View Detailed Results + +[Comprehensive details, logs, traces] + +
+ +
+View All Warnings + +[Minor issues and potential problems] + +
+ +### Recommendations +[Actionable next steps - keep visible] +``` + +## Workflow Run References + +- Format run IDs as links: `[§12345](https://github.com/owner/repo/actions/runs/12345)` +- Include up to 3 most relevant run URLs at end under `**References:**` +- Do NOT add footer attribution (system adds automatically) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cef74aafb4b0..6ffb926f6fa0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -31,6 +31,8 @@ updates: github-actions-dependencies: patterns: - "*" + ignore: + - dependency-name: "github/gh-aw-actions/**" # Managed by gh aw compile. Version-locked to the gh-aw compiler; do not bump. cooldown: default-days: 7 - package-ecosystem: "maven" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c33a1313436..4eb508f5f6f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,40 +16,27 @@ # under the License. name: Build - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v6 + - name: Setup Environment + uses: ./.github/actions/setup-env with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Env details run: | uname -a @@ -60,9 +47,8 @@ jobs: free -m nproc git status - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Noredist Build run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df60179ceb51..0815b4558476 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,21 +16,56 @@ # under the License. name: Simulator CI - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: if: github.repository == 'apache/cloudstack' runs-on: ubuntu-24.04 - + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + with: + install-python: 'true' + install-apt-deps: 'true' + - name: Env details + run: | + uname -a + whoami + javac -version + mvn -v + python3 --version + free -m + nproc + git status + ipmitool -V + - name: Build with Maven + run: | + mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) + - name: Archive artifacts + run: | + mkdir -p /tmp/artifacts + tar -czf /tmp/artifacts/targets.tar.gz $(find . -name "target" -type d) tools/marvin/dist engine/schema/dist utils/conf + tar -czf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository org/apache/cloudstack + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + test: + needs: build + if: github.repository == 'apache/cloudstack' + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -215,30 +250,16 @@ jobs: smoke/test_list_service_offerings smoke/test_list_storage_pools smoke/test_list_volumes"] - steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Setup IPMI Tool for CloudStack run: | # Create cloudstack-common directory if it doesn't exist @@ -256,55 +277,43 @@ jobs: /usr/share/cloudstack-common/ipmitool -C3 $@ EOF sudo chmod 755 /usr/bin/ipmitool - - name: Install Python dependencies run: | python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose - - name: Install jacoco dependencies run: | wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip unzip jacoco-0.8.10.zip -d jacoco - - - name: Env details - run: | - uname -a - whoami - javac -version - mvn -v - python3 --version - free -m - nproc - git status - ipmitool -V - - name: Setup MySQL Server run: | # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#mysql sudo apt-get install -y mysql-server sudo systemctl start mysql - sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; FLUSH PRIVILEGES;" + sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH caching_sha2_password BY ''; FLUSH PRIVILEGES;" sudo systemctl restart mysql sudo mysql -uroot -e "SELECT VERSION();" - - - name: Build with Maven + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + - name: Extract artifacts run: | - mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) - + tar -xzf /tmp/artifacts/targets.tar.gz + mkdir -p ~/.m2/repository + tar -xzf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository - name: Setup Simulator Prerequisites run: | sudo python3 -m pip install --upgrade netaddr mysql-connector-python python3 -m pip install --user --upgrade tools/marvin/dist/[mM]arvin-*.tar.gz mvn -q -Pdeveloper -pl developer -Ddeploydb mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator - - name: Generate jacoco-coverage.sh run: | echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh echo "--xml jacoco-coverage.xml" >> jacoco-report.sh - - name: Start CloudStack Management Server with Simulator run: | export MAVEN_OPTS="-Xmx4096m -XX:MaxMetaspaceSize=800m -Djava.security.egd=file:/dev/urandom -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" @@ -315,7 +324,6 @@ jobs: set -e echo -e "\nStarting Advanced Zone DataCenter deployment" python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true - - name: Run Integration Tests with Simulator run: | mkdir -p integration-test-results/smoke/misc @@ -335,13 +343,12 @@ jobs: bash jacoco-report.sh mvn -Dsimulator -pl client jetty:stop 2>&1 find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \; - - name: Integration Tests Result run: | echo -e "Simulator CI Test Results: (only failures listed)\n" python3 ./tools/marvin/xunit-reader.py integration-test-results/ - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: files: jacoco-coverage.xml fail_ci_if_error: true diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 88b10ac9178f..000000000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Coverage Check - -on: [pull_request, push] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - if: github.repository == 'apache/cloudstack' - name: codecov - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Build CloudStack with Quality Checks - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc) - - - uses: codecov/codecov-action@v4 - with: - files: ./client/target/site/jacoco-aggregate/jacoco.xml - fail_ci_if_error: true - flags: unittests - verbose: true - name: codecov - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 31a8746b85af..a2d581dcac20 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,14 +35,16 @@ jobs: language: ["actions"] steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: category: "Security" diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml index 1d7e7eecd14d..7c3d20a166ea 100644 --- a/.github/workflows/daily-repo-status.lock.yml +++ b/.github/workflows/daily-repo-status.lock.yml @@ -1,393 +1,598 @@ -# -# ___ _ +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"ae44897dc32d20c421588b31a279653abf436964ad86c58885e0edbfbb2f0416","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"46d564922b082d0db93244972e8005ea6904ee5f","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55","digest":"sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55","digest":"sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55","digest":"sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19","digest":"sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} +# ___ _ _ # / _ \ | | (_) -# | |_| | __ _ ___ _ __ | |_ _ +# | |_| | __ _ ___ _ __ | |_ _ ___ # | _ |/ _` |/ _ \ '_ \| __| |/ __| -# | | | | (_| | __/ | | | |_| | ( +# | | | | (_| | __/ | | | |_| | (__ # \_| |_/\__, |\___|_| |_|\__|_|\___| # __/ | # _ _ |___/ # | | | | / _| | -# | | | | ___ _ __ _ __| |_| | _____ +# | | | | ___ _ __ _ __| |_| | _____ ____ # | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.45.0). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. # -# To update this file, edit githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87 and run: -# gh aw +# To update this file, edit githubnext/agentics/workflows/repo-status.md@main and run: +# gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# This workflow creates daily repo status reports. It gathers recent -# activity (issues, PRs, discussions, releases, code changes) and +# This workflow creates daily repo status reports. It gathers recent repository +# activity (issues, PRs, discussions, releases, code changes) and generates # engaging GitHub issues with productivity insights, community highlights, # and project recommendations. # -# Source: githubnext/agentics/workflows/daily-repo-status.md@ +# Source: githubnext/agentics/workflows/repo-status.md@main +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 # -# frontmatter-hash: +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca +# - ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 -name: "Daily Repo Status" -"on": +name: "Repo Status" +on: schedule: - - cron: "25 18 * * *" + - cron: "11 19 * * *" # Friendly format: daily (scattered) workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string permissions: {} concurrency: group: "gh-aw-${{ github.workflow }}" -run-name: "Daily Repo Status" +run-name: "Repo Status" jobs: activation: - runs-on: ubuntu- + runs-on: ubuntu-slim permissions: - contents: + actions: read + contents: read outputs: comment_id: "" comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Repo Status" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_FRONTMATTER_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - destination: /opt/gh-aw/ - - name: Check workflow file - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "daily-repo-status.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + Tools: create_issue, missing_tool, missing_data, noop + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_603e0a48f86a9470_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_603e0a48f86a9470_EOF' + + {{#runtime-import .github/workflows/daily-repo-status.md}} + GH_AW_PROMPT_603e0a48f86a9470_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 agent: - needs: - runs-on: ubuntu- + needs: activation + runs-on: ubuntu-latest permissions: - contents: - issues: - pull-requests: + contents: read + issues: read + pull-requests: read concurrency: group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" - GH_AW_ASSETS_MAX_SIZE_KB: - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/ - GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs. - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config. - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools. - GH_AW_WORKFLOW_ID_SANITIZED: + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_WORKFLOW_ID_SANITIZED: dailyrepostatus outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} has_patch: ${{ steps.collect_output.outputs.has_patch }} - model: ${{ steps.generate_aw_info.outputs.model }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} output_types: ${{ steps.collect_output.outputs.output_types }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - persist-credentials: - - name: Create gh-aw temp - run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir. - - name: Configure Git + persist-credentials: false + - name: Create gh-aw temp directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - # Re-authenticate git with GitHub + git config --global am.keepcr true + # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Checkout PR - id: checkout- + - name: Checkout PR branch + id: checkout-pr if: | - github.event. - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + github.event.pull_request || github.event.issue.pull_request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); await main(); - - name: Generate agentic run - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Parse integrity filter lists + id: parse-guard-vars + env: + GH_AW_BLOCKED_USERS_VAR: ${{ vars.GH_AW_GITHUB_BLOCKED_USERS || '' }} + GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} + GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - script: | - const fs = require('fs'); - - const awInfo = { - engine_id: "copilot", - engine_name: "GitHub Copilot CLI", - model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", - version: "", - agent_version: "0.0.410", - cli_version: "v0.45.0", - workflow_name: "Daily Repo Status", - experimental: false, - supports_tools_allowlist: true, - supports_http_transport: true, - run_id: context.runId, - run_number: context.runNumber, - run_attempt: process.env.GITHUB_RUN_ATTEMPT, - repository: context.repo.owner + '/' + context.repo.repo, - ref: context.ref, - sha: context.sha, - actor: context.actor, - event_name: context.eventName, - staged: false, - allowed_domains: ["defaults"], - firewall_enabled: true, - awf_version: "v0.18.0", - awmg_version: "v0.1.4", - steps: { - firewall: "squid" - }, - created_at: new Date().toISOString() - }; - - // Write to /tmp/gh-aw directory to avoid inclusion in - const tmpPath = '/tmp/gh-aw/aw_info.json'; - fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); - console.log('Generated aw_info.json at:', tmpPath); - console.log(JSON.stringify(awInfo, null, 2)); - - // Set model as output for reuse in other steps/ - core.setOutput('model', awInfo.model); - - name: Validate COPILOT_GITHUB_TOKEN - id: validate- - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot- + name: activation + path: /tmp/gh-aw + - name: Restore agent config folders from base branch + if: steps.checkout-pr.outcome == 'success' env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Install GitHub Copilot - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0. - - name: Install awf - run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18. - - name: Download container - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.18.0 ghcr.io/github/gh-aw-firewall/squid:0.18.0 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts- - - name: Write Safe Outputs + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config run: | - mkdir -p /opt/gh-aw/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/mcp-logs/ - cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"create_issue":{"max":1},"missing_data":{},"missing_tool":{},"noop":{"max":1}} - - cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' - [ + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_9b6646f0d620a78f_EOF' + {"create_issue":{"close_older_issues":true,"labels":["report","daily-status"],"max":1,"title_prefix":"[repo-status] "},"create_report_incomplete_issue":{},"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_9b6646f0d620a78f_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | { - "description": "Create a new GitHub issue for tracking bugs, feature requests, or tasks. Use this for actionable work items that need assignment, labeling, and status tracking. For reports, announcements, or status updates that don't require task tracking, use create_discussion instead. CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [report daily-status] will be automatically added.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "description_suffixes": { + "create_issue": " CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[repo-status] \". Labels [\"report\" \"daily-status\"] will be automatically added." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "create_issue": { + "defaultMax": 1, + "fields": { "body": { - "description": "Detailed issue description in Markdown. Do NOT repeat the title as a heading since it already appears as the issue's h1. Include context, reproduction steps, or acceptance criteria as appropriate.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 }, - "labels": { - "description": "Labels to categorize the issue (e.g., 'bug', 'enhancement'). Labels must exist in the repository.", - "items": { - "type": "string" - }, + "fields": { "type": "array" }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, "parent": { - "description": "Parent issue number for creating sub-issues. This is the numeric ID from the GitHub URL (e.g., 42 in github.com/owner/repo/issues/42). Can also be a temporary_id (e.g., 'aw_abc123', 'aw_Test123') from a previously created issue in the same workflow run.", - "type": [ - "number", - "string" - ] + "issueOrPRNumber": true + }, + "repo": { + "type": "string", + "maxLength": 256 }, "temporary_id": { - "description": "Unique temporary identifier for referencing this issue before it's created. Format: 'aw_' followed by 3 to 8 alphanumeric characters (e.g., 'aw_abc1', 'aw_Test123'). Use '#aw_ID' in body text to reference other issues by their temporary_id; these are replaced with actual issue numbers after creation.", - "pattern": "^aw_[A-Za-z0-9]{4,8}$", "type": "string" }, "title": { - "description": "Concise issue title summarizing the bug, feature, or task. The title appears as the main heading, so keep it brief and descriptive.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "title", - "body" - ], - "type": "object" + } }, - "name": "create_issue" - }, - { - "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "missing_data": { + "defaultMax": 20, + "fields": { "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 }, "reason": { - "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 }, "tool": { - "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "reason" - ], - "type": "object" + } }, - "name": "missing_tool" - }, - { - "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "noop": { + "defaultMax": 1, + "fields": { "message": { - "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 } - }, - "required": [ - "message" - ], - "type": "object" + } }, - "name": "noop" - }, - { - "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" - }, - "context": { - "description": "Additional context about the missing data or where it should come from (max 256 characters).", - "type": "string" - }, - "data_type": { - "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", - "type": "string" + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "reason": { - "description": "Explanation of why this data is needed to complete the task (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 } - }, - "required": [], - "type": "object" - }, - "name": "missing_data" - } - ] - - cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' - { - "create_issue": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - }, - "labels": { - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": - }, - "parent": { - "issueOrPRNumber": - }, - "repo": { - "type": "string", - "maxLength": - }, - "temporary_id": { - "type": "string" - }, - "title": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": } } } - } - - - name: Generate Safe Outputs MCP Server - id: safe-outputs- + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config run: | # Generate a secure random API key (360 bits of entropy, 40+ chars) - # Mask immediately to prevent timing + # Mask immediately to prevent timing vulnerabilities API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${API_KEY}" - PORT= + PORT=3001 - # Set outputs for next + # Set outputs for next steps { echo "safe_outputs_api_key=${API_KEY}" echo "safe_outputs_port=${PORT}" @@ -395,61 +600,84 @@ jobs: echo "Safe Outputs MCP server will run on port ${PORT}" - - name: Start Safe Outputs MCP HTTP - id: safe-outputs- + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start env: DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools. - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config. - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/ + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs run: | - # Environment variables are set above to prevent template - export - export - export - export - export - export + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR - bash /opt/gh-aw/actions/start_safe_outputs_server. + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - - name: Start MCP - id: start-mcp- + - name: Start MCP Gateway + id: start-mcp-gateway env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} run: | - set -eo - mkdir -p /tmp/gh-aw/mcp- + set -eo pipefail + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" - # Export gateway environment variables for MCP config and gateway - export MCP_GATEWAY_PORT="80" + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="8080" export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${MCP_GATEWAY_API_KEY}" - export + export MCP_GATEWAY_API_KEY export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' - mkdir -p /home/runner/. - cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway. + mkdir -p /home/runner/.copilot + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_d075a7f45ab51044_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", "env": { + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "approval-labels": ${{ steps.parse-guard-vars.outputs.approval_labels }}, + "blocked-users": ${{ steps.parse-guard-vars.outputs.blocked_users }}, + "min-integrity": "none", + "repos": "all", + "trusted-users": ${{ steps.parse-guard-vars.outputs.trusted_users }} + } } }, "safeoutputs": { @@ -457,6 +685,13 @@ jobs: "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", "headers": { "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } } } }, @@ -467,212 +702,113 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - - - name: Generate workflow - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - with: - script: | - const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); - await generateWorkflowOverview(core); - - name: Create prompt with built-in + GH_AW_MCP_CONFIG_d075a7f45ab51044_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - bash /opt/gh-aw/actions/create_prompt_first. - cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" - - - cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GitHub API Access Instructions - - The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. - - - To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. - - Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). - - **IMPORTANT - temporary_id format rules:** - - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) - - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/ - - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) - - Valid alphanumeric characters: - - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) - - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, - - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto- - - Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. - - Discover available tools from the safeoutputs MCP server. - - **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. - - **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. - - - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: # - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: - {{/if}} - - - - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/workflows/daily-repo-status.md}} - - - name: Substitute - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env. - } - }); - - name: Interpolate variables and render - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); await main(); - - name: Validate prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - run: bash /opt/gh-aw/actions/validate_prompt_placeholders. - - name: Print - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - run: bash /opt/gh-aw/actions/print_prompt_summary. - - name: Clean git - run: bash /opt/gh-aw/actions/clean_git_credentials. - - name: Execute GitHub Copilot - id: + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" + - name: Execute GitHub Copilot CLI + id: agentic_execution # Copilot CLI tool arguments (sorted): - timeout-minutes: + timeout-minutes: 20 run: | - set -o - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.18.0 --skip-pull \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio. + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55,squid=sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca,agent=sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731,api-proxy=sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: - COPILOT_AGENT_RUNNER_TYPE: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config. - GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} - XDG_CONFIG_HOME: /home/ - - name: Configure Git + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - # Re-authenticate git with GitHub + git config --global am.keepcr true + # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Copy Copilot session state files to + - name: Copy Copilot session state files to logs if: always() - continue-on-error: - run: | - # Copy Copilot session state files to logs folder for artifact - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || - echo "Session state files copied successfully" - - echo "No session-state directory found at $SESSION_STATE_DIR" - - - name: Stop MCP + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" + - name: Stop MCP Gateway if: always() - continue-on-error: + continue-on-error: true env: MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | - bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" - - name: Redact secrets in + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" + - name: Redact secrets in logs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -680,343 +816,579 @@ jobs: SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload Safe + - name: Append agent step summary if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. - with: - name: safe- - path: ${{ env.GH_AW_SAFE_OUTPUTS }} - if-no-files-found: - - name: Ingest agent - id: + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_GITHUB_REFS: "" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); await main(); - - name: Upload sanitized agent - if: always() && env. - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. - with: - name: agent- - path: ${{ env.GH_AW_AGENT_OUTPUT }} - if-no-files-found: - - name: Upload engine output - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. - with: - name: - path: | - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls. - if-no-files-found: - - name: Parse agent logs for step + - name: Parse agent logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); - - name: Parse MCP Gateway logs for step + - name: Parse MCP Gateway logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); await main(); - - name: Print firewall + - name: Print firewall logs if: always() - continue-on-error: + continue-on-error: true env: - AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/ + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs run: | - # Fix permissions on firewall logs so they can be uploaded as - # AWF runs with sudo, creating files owned by - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) - if command -v awf &> /dev/null; + if command -v awf &> /dev/null; then awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" - + else echo 'AWF binary not installed, skipping firewall log summary' - - - name: Upload agent + fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary if: always() - continue-on-error: - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - name: agent- + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent path: | - /tmp/gh-aw/aw-prompts/prompt. - /tmp/gh-aw/aw_info. + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ - /tmp/gh-aw/agent-stdio. + /tmp/gh-aw/proxy-logs/ + !/tmp/gh-aw/proxy-logs/proxy-tls/ + /tmp/gh-aw/agent_usage.json + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ - if-no-files-found: + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json + if-no-files-found: ignore conclusion: needs: - - - - - - - - - if: (always()) && (needs.agent.result != 'skipped') - runs-on: ubuntu- + - activation + - agent + - detection + - safe_outputs + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') + runs-on: ubuntu-slim permissions: - contents: - issues: + contents: read + issues: write + concurrency: + group: "gh-aw-conclusion-daily-repo-status" + cancel-in-progress: false + queue: max outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/safeoutputs/ - - name: Setup agent output environment + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f - - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process No-Op - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/noop.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); await main(); - - name: Record Missing - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); await main(); - - name: Handle Agent - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "daily-repo-status" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} - GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + - name: Record missing tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); await main(); - - name: Handle No-Op - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # - env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); + await main(); + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" + GH_AW_WORKFLOW_ID: "daily-repo-status" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "20" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); await main(); detection: needs: - if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' - runs-on: ubuntu- - permissions: {} - concurrency: - group: "gh-aw-copilot-${{ github.workflow }}" - timeout-minutes: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') + runs-on: ubuntu-latest + permissions: + contents: read outputs: - success: ${{ steps.parse_results.outputs.success }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/threat-detection/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - name: agent- - path: /tmp/gh-aw/threat-detection/ - - name: Echo agent output + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca + - name: Check if detection needed + id: detection_guard + if: always() env: - AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - echo "Agent output-types: $AGENT_OUTPUT_TYPES" - - name: Setup threat - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - WORKFLOW_NAME: "Daily Repo Status" + WORKFLOW_NAME: "Repo Status" WORKFLOW_DESCRIPTION: "This workflow creates daily repo status reports. It gathers recent repository\nactivity (issues, PRs, discussions, releases, code changes) and generates\nengaging GitHub issues with productivity insights, community highlights,\nand project recommendations." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); await main(); - - name: Ensure threat-detection directory and + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - mkdir -p /tmp/gh-aw/threat- - touch /tmp/gh-aw/threat-detection/detection. - - name: Validate COPILOT_GITHUB_TOKEN - id: validate- - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot- + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Install GitHub Copilot - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0. - - name: Execute GitHub Copilot - id: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool shell(cat) - # --allow-tool shell(grep) - # --allow-tool shell(head) - # --allow-tool shell(jq) - # --allow-tool shell(ls) - # --allow-tool shell(tail) - # --allow-tool shell(wc) - timeout-minutes: + timeout-minutes: 20 run: | - set -o - COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" - mkdir -p /tmp/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/agent/ - mkdir -p /tmp/gh-aw/sandbox/agent/logs/ - copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection. + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: - COPILOT_AGENT_RUNNER_TYPE: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} - XDG_CONFIG_HOME: /home/ - - name: Parse threat detection - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - - name: Upload threat detection + name: detection + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" with: - name: threat-detection. - path: /tmp/gh-aw/threat-detection/detection. - if-no-files-found: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } safe_outputs: needs: - - - - - if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') - runs-on: ubuntu- + - activation + - agent + - detection + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' + runs-on: ubuntu-slim permissions: - contents: - issues: - timeout-minutes: + contents: read + issues: write + timeout-minutes: 15 env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/daily-repo-status" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" GH_AW_WORKFLOW_ID: "daily-repo-status" - GH_AW_WORKFLOW_NAME: "Daily Repo Status" - GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/tree/d19056381ba48cb1f7c78510c23069701fa7ae87/workflows/daily-repo-status.md" + GH_AW_WORKFLOW_NAME: "Repo Status" + GH_AW_WORKFLOW_SOURCE: "githubnext/agentics/workflows/repo-status.md@main" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/githubnext/agentics/blob/main/workflows/repo-status.md" outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + created_issue_number: ${{ steps.process_safe_outputs.outputs.created_issue_number }} + created_issue_url: ${{ steps.process_safe_outputs.outputs.created_issue_url }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/ - - name: Download agent output - continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Repo Status" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-repo-status.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent- - path: /tmp/gh-aw/safeoutputs/ - - name: Setup agent output environment + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f - - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process Safe - id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"create_report_incomplete_issue\":{},\"mentions\":{\"enabled\":false},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore diff --git a/.github/workflows/daily-repo-status.md b/.github/workflows/daily-repo-status.md index 431b4afb91a6..8debf5708437 100644 --- a/.github/workflows/daily-repo-status.md +++ b/.github/workflows/daily-repo-status.md @@ -22,15 +22,19 @@ tools: # reading issues, pull requests and comments from 3rd-parties # If in a private repo this has no particular effect. lockdown: false + min-integrity: none # This workflow is allowed to examine and comment on any issues safe-outputs: + mentions: false + allowed-github-references: [] create-issue: title-prefix: "[repo-status] " labels: [report, daily-status] -source: githubnext/agentics/workflows/daily-repo-status.md@d19056381ba48cb1f7c78510c23069701fa7ae87 + close-older-issues: true +source: githubnext/agentics/workflows/repo-status.md@main --- -# Daily Repo Status +# Repo Status Create an upbeat daily status report for the repo as a GitHub issue. diff --git a/.github/workflows/docker-cloudstack-simulator.yml b/.github/workflows/docker-cloudstack-simulator.yml index 8d23ac449dd5..dc00968563d0 100644 --- a/.github/workflows/docker-cloudstack-simulator.yml +++ b/.github/workflows/docker-cloudstack-simulator.yml @@ -35,10 +35,10 @@ concurrency: jobs: build: if: github.repository == 'apache/cloudstack' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Login to Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKERHUB_USER }} @@ -47,7 +47,9 @@ jobs: - name: Set Docker repository name run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set ACS version run: echo "ACS_VERSION=$(grep '' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 2410f7b9e457..3acf91dcc1ef 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -1,4 +1,5 @@ -# +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"3cfe810a6f1f402ae902098f1dc14b99a4281a5986d58b76e96eec1a2fde7646","compiler_version":"v0.76.1","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"46d564922b082d0db93244972e8005ea6904ee5f","version":"v0.76.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55","digest":"sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55","digest":"sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55","digest":"sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.19","digest":"sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f"},{"image":"ghcr.io/github/github-mcp-server:v1.0.4","digest":"sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) # | |_| | __ _ ___ _ __ | |_ _ ___ @@ -13,28 +14,56 @@ # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.45.0). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.76.1). DO NOT EDIT. # -# To update this file, edit github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 and run: +# To update this file, edit github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 and run: # gh aw compile # Not all edits will cause changes to this file. # # For more information: https://github.github.com/gh-aw/introduction/overview/ # # -# Source: github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +# Source: github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 # # Resolved workflow manifest: # Imports: -# - github/gh-aw/.github/workflows/shared/reporting.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +# - github/gh-aw/.github/workflows/shared/reporting.md@359795d49ada21681ab616bd4cbcb144a7387115 +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 # -# frontmatter-hash: 7bc83974fa1e47c12b40c3333872e4126711d5c6624022cc78b76047289d8b63 +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 +# - ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca +# - ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f +# - ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 +# - node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 name: "Issue Triage Agent" -"on": +on: schedule: - - cron: "0 14 * * 1-5" + - cron: "49 14 * * 1-5" + # Friendly format: daily around 14:00 on weekdays (scattered) workflow_dispatch: + inputs: + aw_context: + default: "" + description: "Agent caller context (used internally by Agentic Workflows)." + required: false + type: string permissions: {} @@ -47,324 +76,503 @@ jobs: activation: runs-on: ubuntu-slim permissions: + actions: read contents: read outputs: comment_id: "" comment_repo: "" + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions - - name: Check workflow file timestamps - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AGENT_VERSION: "1.0.52" + GH_AW_INFO_CLI_VERSION: "v0.76.1" + GH_AW_INFO_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_FRONTMATTER_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_FRONTMATTER_EMOJI: "🔧" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + .antigravity + .claude + .codex + .crush + .gemini + .opencode + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" + GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "issue-triage-agent.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.76.1" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" + { + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + GH_AW_PROMPT_9a8f42497f411c95_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + Tools: add_comment, add_labels, missing_tool, missing_data, noop + + GH_AW_PROMPT_9a8f42497f411c95_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + The following GitHub context information is available for this workflow: + {{#if github.actor}} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if github.repository}} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if github.workspace}} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} + - **issue-number**: #__GH_AW_EXPR_802A9F6A__ + {{/if}} + {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} + - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ + {{/if}} + {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} + - **pull-request-number**: #__GH_AW_EXPR_463A214A__ + {{/if}} + {{#if github.event.comment.id || github.aw.context.comment_id}} + - **comment-id**: __GH_AW_EXPR_FF1D34CE__ + {{/if}} + {{#if github.run_id}} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_9a8f42497f411c95_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" + cat << 'GH_AW_PROMPT_9a8f42497f411c95_EOF' + + {{#runtime-import .github/aw/imports/github/gh-aw/359795d49ada21681ab616bd4cbcb144a7387115/.github_workflows_shared_reporting.md}} + {{#runtime-import .github/workflows/shared/noop-reminder.md}} + {{#runtime-import .github/workflows/issue-triage-agent.md}} + GH_AW_PROMPT_9a8f42497f411c95_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 agent: needs: activation runs-on: ubuntu-latest permissions: - contents: read - issues: read + issues: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json GH_AW_WORKFLOW_ID_SANITIZED: issuetriageagent outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + effective_tokens_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.effective_tokens_rate_limit_error || 'false' }} has_patch: ${{ steps.collect_output.outputs.has_patch }} - model: ${{ steps.generate_aw_info.outputs.model }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} output_types: ${{ steps.collect_output.outputs.output_types }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + run: | + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Create gh-aw temp directory - run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git config --global am.keepcr true # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - - name: Generate agentic run info - id: generate_aw_info - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const fs = require('fs'); - - const awInfo = { - engine_id: "copilot", - engine_name: "GitHub Copilot CLI", - model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", - version: "", - agent_version: "0.0.410", - cli_version: "v0.45.0", - workflow_name: "Issue Triage Agent", - experimental: false, - supports_tools_allowlist: true, - supports_http_transport: true, - run_id: context.runId, - run_number: context.runNumber, - run_attempt: process.env.GITHUB_RUN_ATTEMPT, - repository: context.repo.owner + '/' + context.repo.repo, - ref: context.ref, - sha: context.sha, - actor: context.actor, - event_name: context.eventName, - staged: false, - allowed_domains: ["defaults"], - firewall_enabled: true, - awf_version: "v0.18.0", - awmg_version: "v0.1.4", - steps: { - firewall: "squid" - }, - created_at: new Date().toISOString() - }; - - // Write to /tmp/gh-aw directory to avoid inclusion in PR - const tmpPath = '/tmp/gh-aw/aw_info.json'; - fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); - console.log('Generated aw_info.json at:', tmpPath); - console.log(JSON.stringify(awInfo, null, 2)); - - // Set model as output for reuse in other steps/jobs - core.setOutput('model', awInfo.model); - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - name: Install GitHub Copilot CLI - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.410 - - name: Install awf binary - run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18.0 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 - name: Determine automatic lockdown mode for GitHub MCP Server id: determine-automatic-lockdown - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) env: GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} with: script: | - const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); await determineAutomaticLockdown(github, context, core); + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.18.0 ghcr.io/github/gh-aw-firewall/squid:0.18.0 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine - - name: Write Safe Outputs Config + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca ghcr.io/github/gh-aw-mcpg:v0.3.19@sha256:a6c890d7c24d7190c9ef97b9c954cc4cffaae6b01c371ced1f959f1370b1f68f ghcr.io/github/github-mcp-server:v1.0.4@sha256:e3816a476a977cfb836e7d221510011436c654d11861db66ecfd826601aba6a4 node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14 + - name: Generate Safe Outputs Config run: | - mkdir -p /opt/gh-aw/safeoutputs + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' - {"add_comment":{"max":1},"add_labels":{"allowed":["bug","feature","enhancement","documentation","question","help-wanted","good-first-issue"],"max":3},"missing_data":{},"missing_tool":{},"noop":{"max":1}} - GH_AW_SAFE_OUTPUTS_CONFIG_EOF - cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' - [ + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_40bb017992a44aa9_EOF' + {"add_comment":{"max":1},"add_labels":{"allowed":["bug","feature","enhancement","documentation","question","help-wanted","good-first-issue"]},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_40bb017992a44aa9_EOF + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | + { + "description_suffixes": { + "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added. Supports reply_to_id for discussion threading.", + "add_labels": " CONSTRAINTS: Only these labels are allowed: [\"bug\" \"feature\" \"enhancement\" \"documentation\" \"question\" \"help-wanted\" \"good-first-issue\"]." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | { - "description": "Add a comment to an existing GitHub issue, pull request, or discussion. Use this to provide feedback, answer questions, or add information to an existing conversation. For creating new items, use create_issue, create_discussion, or create_pull_request instead. IMPORTANT: Comments are subject to validation constraints enforced by the MCP server - maximum 65536 characters for the complete comment (including footer which is added automatically), 10 mentions (@username), and 50 links. Exceeding these limits will result in an immediate error with specific guidance. CONSTRAINTS: Maximum 1 comment(s) can be added.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "add_comment": { + "defaultMax": 1, + "fields": { "body": { - "description": "The comment text in Markdown format. This is the 'body' field - do not use 'comment_body' or other variations. Provide helpful, relevant information that adds value to the conversation. CONSTRAINTS: The complete comment (your body text + automatically added footer) must not exceed 65536 characters total. Maximum 10 mentions (@username), maximum 50 links (http/https URLs). A footer (~200-500 characters) is automatically appended with workflow attribution, so leave adequate space. If these limits are exceeded, the tool call will fail with a detailed error message indicating which constraint was violated.", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "item_number": { - "description": "The issue, pull request, or discussion number to comment on. This is the numeric ID from the GitHub URL (e.g., 123 in github.com/owner/repo/issues/123). If omitted, the tool will attempt to resolve the target from the current workflow context (triggering issue, PR, or discussion).", - "type": "number" + "issueOrPRNumber": true + }, + "reply_to_id": { + "type": "string", + "maxLength": 256 + }, + "repo": { + "type": "string", + "maxLength": 256 } - }, - "required": [ - "body" - ], - "type": "object" + } }, - "name": "add_comment" - }, - { - "description": "Add labels to an existing GitHub issue or pull request for categorization and filtering. Labels must already exist in the repository. For creating new issues with labels, use create_issue with the labels property instead. CONSTRAINTS: Only these labels are allowed: [bug feature enhancement documentation question help-wanted good-first-issue].", - "inputSchema": { - "additionalProperties": false, - "properties": { + "add_labels": { + "defaultMax": 5, + "fields": { "item_number": { - "description": "Issue or PR number to add labels to. This is the numeric ID from the GitHub URL (e.g., 456 in github.com/owner/repo/issues/456). If omitted, adds labels to the item that triggered this workflow.", - "type": "number" + "issueNumberOrTemporaryId": true }, "labels": { - "description": "Label names to add (e.g., ['bug', 'priority-high']). Labels must exist in the repository.", - "items": { - "type": "string" - }, - "type": "array" + "required": true, + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "repo": { + "type": "string", + "maxLength": 256 } - }, - "type": "object" + } }, - "name": "add_labels" - }, - { - "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 512 }, "reason": { - "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 }, "tool": { - "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", - "type": "string" + "type": "string", + "sanitize": true, + "maxLength": 128 } - }, - "required": [ - "reason" - ], - "type": "object" + } }, - "name": "missing_tool" - }, - { - "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", - "inputSchema": { - "additionalProperties": false, - "properties": { + "noop": { + "defaultMax": 1, + "fields": { "message": { - "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 } - }, - "required": [ - "message" - ], - "type": "object" + } }, - "name": "noop" - }, - { - "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "alternatives": { - "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", - "type": "string" - }, - "context": { - "description": "Additional context about the missing data or where it should come from (max 256 characters).", - "type": "string" - }, - "data_type": { - "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", - "type": "string" + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 }, "reason": { - "description": "Explanation of why this data is needed to complete the task (max 256 characters).", - "type": "string" + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 } - }, - "required": [], - "type": "object" - }, - "name": "missing_data" - } - ] - GH_AW_SAFE_OUTPUTS_TOOLS_EOF - cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' - { - "add_comment": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "item_number": { - "issueOrPRNumber": true - } - } - }, - "add_labels": { - "defaultMax": 5, - "fields": { - "item_number": { - "issueOrPRNumber": true - }, - "labels": { - "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": 512 - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 } } } - } - GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + await main(); - name: Generate Safe Outputs MCP Server Config id: safe-outputs-config run: | @@ -387,59 +595,81 @@ jobs: id: safe-outputs-start env: DEBUG: '*' + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs run: | # Environment variables are set above to prevent template injection export DEBUG + export GH_AW_SAFE_OUTPUTS export GH_AW_SAFE_OUTPUTS_PORT export GH_AW_SAFE_OUTPUTS_API_KEY export GH_AW_SAFE_OUTPUTS_TOOLS_PATH export GH_AW_SAFE_OUTPUTS_CONFIG_PATH export GH_AW_MCP_LOG_DIR - bash /opt/gh-aw/actions/start_safe_outputs_server.sh + bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - name: Start MCP Gateway id: start-mcp-gateway env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} - GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} + GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} run: | set -eo pipefail - mkdir -p /tmp/gh-aw/mcp-config + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" # Export gateway environment variables for MCP config and gateway script - export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_PORT="8080" export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_HOST_DOMAIN="localhost" MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') echo "::add-mask::${MCP_GATEWAY_API_KEY}" export MCP_GATEWAY_API_KEY export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + export GH_AW_MCP_CLI_SERVERS='["safeoutputs"]' + echo GH_AW_MCP_CLI_SERVERS='["safeoutputs"]' >> "$GITHUB_ENV" + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + case "${DOCKER_HOST:-}" in + unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; + /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; + * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; + esac + DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.19' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_8874a73d1c6a94b7_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "container": "ghcr.io/github/github-mcp-server:v1.0.4", "env": { - "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "issues,labels" + }, + "guard-policies": { + "allow-only": { + "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", + "repos": "$GITHUB_MCP_GUARD_REPOS" + } } }, "safeoutputs": { @@ -447,6 +677,13 @@ jobs: "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", "headers": { "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ] + } } } }, @@ -457,199 +694,96 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_EOF - - name: Generate workflow overview - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); - await generateWorkflowOverview(core); - - name: Create prompt with built-in context - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - bash /opt/gh-aw/actions/create_prompt_first.sh - cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" - - GH_AW_PROMPT_EOF - cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" - cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GitHub API Access Instructions - - The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. - - - To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. - - Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). - - **IMPORTANT - temporary_id format rules:** - - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) - - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i - - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) - - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 - - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) - - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 - - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate - - Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. - - Discover available tools from the safeoutputs MCP server. - - **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. - - **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. - - - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: __GH_AW_GITHUB_ACTOR__ - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: __GH_AW_GITHUB_REPOSITORY__ - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: __GH_AW_GITHUB_WORKSPACE__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ - {{/if}} - - - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/aw/imports/github/gh-aw/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github_workflows_shared_reporting.md}} - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" - {{#runtime-import .github/workflows/issue-triage-agent.md}} - GH_AW_PROMPT_EOF - - name: Substitute placeholders - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution function - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE - } - }); - - name: Interpolate variables and render templates - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + GH_AW_MCP_CONFIG_8874a73d1c6a94b7_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); await main(); - - name: Validate prompt placeholders - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh - - name: Print prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - run: bash /opt/gh-aw/actions/print_prompt_summary.sh - - name: Clean git credentials - run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): timeout-minutes: 5 run: | set -o pipefail - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.18.0 --skip-pull \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-4.1":["copilot/gpt-4.1*","openai/gpt-4.1*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.55,squid=sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca,agent=sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731,api-proxy=sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: + AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-sonnet-4.6' }} GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json - GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] XDG_CONFIG_HOME: /home/runner + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git config --global am.keepcr true # Re-authenticate git with GitHub token SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" echo "Git configured with standard GitHub Actions identity" - name: Copy Copilot session state files to logs if: always() continue-on-error: true - run: | - # Copy Copilot session state files to logs folder for artifact collection - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; then - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true - echo "Session state files copied successfully" - else - echo "No session-state directory found at $SESSION_STATE_DIR" - fi + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" - name: Stop MCP Gateway if: always() continue-on-error: true @@ -658,15 +792,15 @@ jobs: MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | - bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" - name: Redact secrets in logs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -674,62 +808,51 @@ jobs: SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload Safe Outputs + - name: Append agent step summary if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: safe-output - path: ${{ env.GH_AW_SAFE_OUTPUTS }} - if-no-files-found: warn + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs + if: always() + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true - name: Ingest agent output id: collect_output if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); await main(); - - name: Upload sanitized agent output - if: always() && env.GH_AW_AGENT_OUTPUT - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: agent-output - path: ${{ env.GH_AW_AGENT_OUTPUT }} - if-no-files-found: warn - - name: Upload engine output files - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: agent_outputs - path: | - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls.log - if-no-files-found: ignore - name: Parse agent logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); - name: Parse MCP Gateway logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); await main(); - name: Print firewall logs if: always() @@ -737,28 +860,65 @@ jobs: env: AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs run: | - # Fix permissions on firewall logs so they can be uploaded as artifacts + # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts # AWF runs with sudo, creating files owned by root - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) if command -v awf &> /dev/null; then awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" else echo 'AWF binary not installed, skipping firewall log summary' fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + await main(); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi - name: Upload agent artifacts if: always() continue-on-error: true - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: agent-artifacts + name: agent path: | /tmp/gh-aw/aw-prompts/prompt.txt - /tmp/gh-aw/aw_info.json + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent_usage.json /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json if-no-files-found: ignore conclusion: @@ -767,207 +927,370 @@ jobs: - agent - detection - safe_outputs - if: (always()) && (needs.agent.result != 'skipped') + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.stale_lock_file_failed == 'true') runs-on: ubuntu-slim permissions: contents: read discussions: write issues: write pull-requests: write + concurrency: + group: "gh-aw-conclusion-issue-triage-agent" + cancel-in-progress: false + queue: max outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/safeoutputs/ + name: agent + path: /tmp/gh-aw/ - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - - name: Process No-Op Messages + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Process no-op messages id: noop - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: 1 + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/noop.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); await main(); - - name: Record Missing Tool + - name: Record missing tool id: missing_tool - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); await main(); - - name: Handle Agent Failure - id: handle_agent_failure - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "issue-triage-agent" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); await main(); - - name: Handle No-Op Message - id: handle_noop_message - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" + GH_AW_WORKFLOW_ID: "issue-triage-agent" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ENGINE_ID: "copilot" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_EFFECTIVE_TOKENS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.effective_tokens_rate_limit_error || 'false' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "5" + GH_AW_MAX_EFFECTIVE_TOKENS: "25000000" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); await main(); detection: - needs: agent - if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + needs: + - activation + - agent + if: > + always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') runs-on: ubuntu-latest - permissions: {} - timeout-minutes: 10 + permissions: + contents: read outputs: - success: ${{ steps.parse_results.outputs.success }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 - with: - destination: /opt/gh-aw/actions - - name: Download agent artifacts - continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - name: agent-artifacts - path: /tmp/gh-aw/threat-detection/ + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - name: agent-output - path: /tmp/gh-aw/threat-detection/ - - name: Echo agent output types + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.55@sha256:138c363411decc9a61a5af9b95e8d64c76648b00add0ba06fc7ba786f0e72731 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.55@sha256:4142b873b678cd3279b98dcbe464857d56ea2f2348719b00379cdf35dd843ff3 ghcr.io/github/gh-aw-firewall/squid:0.25.55@sha256:74084b704d8d3664a363655986664d70bd9cdb4830532d0b35cd784d867aabca + - name: Check if detection needed + id: detection_guard + if: always() env: - AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} run: | - echo "Agent output-types: $AGENT_OUTPUT_TYPES" + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + for f in /tmp/gh-aw/aw-*.bundle; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true - name: Setup threat detection - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Issue Triage Agent" WORKFLOW_DESCRIPTION: "No description provided" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); await main(); - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | mkdir -p /tmp/gh-aw/threat-detection touch /tmp/gh-aw/threat-detection/detection.log - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + package-manager-cache: false - name: Install GitHub Copilot CLI - run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.410 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.52 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.55 - name: Execute GitHub Copilot CLI - id: agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + id: detection_agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool shell(cat) - # --allow-tool shell(grep) - # --allow-tool shell(head) - # --allow-tool shell(jq) - # --allow-tool shell(ls) - # --allow-tool shell(tail) - # --allow-tool shell(wc) timeout-minutes: 20 run: | set -o pipefail - COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" - mkdir -p /tmp/ - mkdir -p /tmp/gh-aw/ - mkdir -p /tmp/gh-aw/agent/ - mkdir -p /tmp/gh-aw/sandbox/agent/logs/ - copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.55/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","github.com","host.docker.internal","telemetry.enterprise.githubcopilot.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000},"container":{"imageTag":"0.25.55"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" + fi + # shellcheck disable=SC1003 + sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: + AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.76.1 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] XDG_CONFIG_HOME: /home/runner - - name: Parse threat detection results - id: parse_results - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - name: Upload threat detection log - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: threat-detection.log + name: detection path: /tmp/gh-aw/threat-detection/detection.log if-no-files-found: ignore + - name: Parse and conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + with: + script: | + try { + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + } catch (loadErr) { + const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; + const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; + const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); + core.error(msg); + core.setOutput('reason', 'parse_error'); + if (continueOnError && !detectionExecutionFailed) { + core.warning('\u26A0\uFE0F ' + msg); + core.setOutput('conclusion', 'warning'); + core.setOutput('success', 'false'); + } else { + core.setOutput('conclusion', 'failure'); + core.setOutput('success', 'false'); + core.setFailed(msg); + } + } safe_outputs: needs: + - activation - agent - detection - if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' runs-on: ubuntu-slim permissions: contents: read @@ -976,41 +1299,89 @@ jobs: pull-requests: write timeout-minutes: 15 env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/issue-triage-agent" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} + GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_VERSION: "1.0.52" + GH_AW_WORKFLOW_EMOJI: "🔧" GH_AW_WORKFLOW_ID: "issue-triage-agent" GH_AW_WORKFLOW_NAME: "Issue Triage Agent" - GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4" - GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/tree/94662b1dee8ce96c876ba9f33b3ab8be32de82a4/.github/workflows/issue-triage-agent.md" + GH_AW_WORKFLOW_SOURCE: "github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/github/gh-aw/blob/359795d49ada21681ab616bd4cbcb144a7387115/.github/workflows/issue-triage-agent.md" outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + comment_id: ${{ steps.process_safe_outputs.outputs.comment_id }} + comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + id: setup + uses: github/gh-aw-actions/setup@46d564922b082d0db93244972e8005ea6904ee5f # v0.76.1 with: - destination: /opt/gh-aw/actions + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Issue Triage Agent" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage-agent.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.52" + GH_AW_INFO_AWF_VERSION: "v0.25.55" + GH_AW_INFO_BODY_MODIFIED: "false" + GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact + id: download-agent-output continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent-output - path: /tmp/gh-aw/safeoutputs/ + name: agent + path: /tmp/gh-aw/ - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash run: | - mkdir -p /tmp/gh-aw/safeoutputs/ - find "/tmp/gh-aw/safeoutputs/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" - name: Process Safe Outputs id: process_safe_outputs - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + if-no-files-found: ignore diff --git a/.github/workflows/issue-triage-agent.md b/.github/workflows/issue-triage-agent.md index 06ff227b740d..26c43644618c 100644 --- a/.github/workflows/issue-triage-agent.md +++ b/.github/workflows/issue-triage-agent.md @@ -1,11 +1,11 @@ --- on: - schedule: 0 14 * * 1-5 + schedule: daily around 14:00 on weekdays workflow_dispatch: null permissions: issues: read imports: -- github/gh-aw/.github/workflows/shared/reporting.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +- github/gh-aw/.github/workflows/shared/reporting.md@359795d49ada21681ab616bd4cbcb144a7387115 safe-outputs: add-comment: {} add-labels: @@ -17,10 +17,12 @@ safe-outputs: - question - help-wanted - good-first-issue -source: github/gh-aw/.github/workflows/issue-triage-agent.md@94662b1dee8ce96c876ba9f33b3ab8be32de82a4 +emoji: 🔧 +source: github/gh-aw/.github/workflows/issue-triage-agent.md@359795d49ada21681ab616bd4cbcb144a7387115 strict: true timeout-minutes: 5 tools: + cli-proxy: true github: toolsets: - issues @@ -45,7 +47,7 @@ Hi @{author}! I've categorized this issue as **{label_name}** based on the follo **Reasoning**: {brief_explanation_of_why_this_label}
-View Triage Details +View Triage Details #### Analysis - **Keywords detected**: {list_of_keywords_that_matched} @@ -76,3 +78,15 @@ For efficiency, if multiple issues are triaged in a single run: 3. Optionally: Create a discussion summarizing all triage actions for that run This provides both per-issue context and batch visibility. + +## Labels + +- `bug`: Indicates a problem or error in the code that needs fixing. +- `feature`: Represents a new feature request or enhancement to existing functionality. +- `enhancement`: Suggests improvements to existing features or code. +- `documentation`: Pertains to issues related to documentation, such as missing or unclear docs. +- `question`: Used for issues that are asking for clarification or have questions about the project. +- `help-wanted`: Indicates that the issue is a good candidate for external contributions and help +- `good-first-issue`: Marks issues that are suitable for newcomers to the project, often with simpler scope. + +{{#runtime-import shared/noop-reminder.md}} diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 7ccd6600ab97..3d510fa5dc34 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -15,54 +15,51 @@ # specific language governing permissions and limitations # under the License. -name: Main Branch Sonar Quality Check - +name: Sonar Quality Check (Main) +permissions: + contents: read on: push: branches: - main - -permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: if: github.repository == 'apache/cloudstack' - name: Main Sonar JaCoCo Build - runs-on: ubuntu-22.04 + name: Sonar JaCoCo Coverage + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - - - name: Run Tests with Coverage + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss + - name: Run Build and Tests with Coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack + run: mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.branch.name=${{ github.ref_name }} + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/merge-conflict-checker.yml b/.github/workflows/merge-conflict-checker.yml index a997cb94ccc0..f23719f7183a 100644 --- a/.github/workflows/merge-conflict-checker.yml +++ b/.github/workflows/merge-conflict-checker.yml @@ -17,28 +17,26 @@ name: "PR Merge Conflict Check" on: - push: - pull_request: - types: [opened, synchronize, reopened] + schedule: + - cron: '*/10 * * * *' + workflow_dispatch: -permissions: # added using https://github.com/step-security/secure-workflows - contents: read +permissions: {} concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: "gh-aw-${{ github.workflow }}" jobs: triage: permissions: - pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs - runs-on: ubuntu-22.04 + pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs + runs-on: ubuntu-24.04 steps: - - name: Conflict Check - uses: eps1lon/actions-label-merge-conflict@v2.0.0 - with: - repoToken: "${{ secrets.GITHUB_TOKEN }}" - dirtyLabel: "status:has-conflicts" - removeOnDirtyLabel: "status:ready-for-review" - continueOnMissingPermissions: true - commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." + - name: Conflict Check + uses: eps1lon/actions-label-merge-conflict@1df065ebe6e3310545d4f4c4e862e43bdca146f0 # v3.0.3 + with: + repoToken: "${{ secrets.GITHUB_TOKEN }}" + dirtyLabel: "status:has-conflicts" + removeOnDirtyLabel: "status:ready-for-review" + continueOnMissingPermissions: true + commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 895a597659de..1ea00be9112f 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -29,17 +29,23 @@ concurrency: jobs: pre-commit: name: Run pre-commit - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Out - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + cache: 'pip' - name: Install - run: | - python -m pip install --upgrade pip - pip install pre-commit + run: pip install pre-commit - name: Set PY run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v5 + - name: Cache pre-commit environments + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index 21b8e197d825..a36a956856e7 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -16,32 +16,27 @@ # under the License. name: License Check - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - java-version: '17' - distribution: 'adopt' - architecture: x64 - cache: maven + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: RAT licence checks run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check - name: Rat Report if: always() diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 9f5c3a194bc7..e614d1864824 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -16,58 +16,52 @@ # under the License. name: Sonar Quality Check - -on: [pull_request] - permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - + contents: read + pull-requests: write +on: + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: - if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository name: Sonar JaCoCo Coverage - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: "refs/pull/${{ github.event.number }}/merge" fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Run Build and Tests with Coverage - id: coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube + if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} PR_ID: ${{ github.event.pull_request.number }} HEADREF: ${{ github.event.pull_request.head.ref }} run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 842e4497a4ad..c0da5f98cc8e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: stale-issue-message: 'This issue is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' stale-pr-message: 'This PR is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' @@ -41,7 +41,7 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' - - uses: actions/stale@v10 + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: stale-issue-label: 'archive' days-before-stale: 240 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 4580b6bbd5da..0a6cd5f3ea5f 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -28,15 +28,19 @@ permissions: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Node - uses: actions/setup-node@v5 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 16 + cache: 'npm' + cache-dependency-path: 'ui/package-lock.json' - name: Env details run: | @@ -55,7 +59,7 @@ jobs: npm run lint npm run test:unit - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 if: github.repository == 'apache/cloudstack' with: working-directory: ui diff --git a/README.md b/README.md index a5aacb49f6b5..852674ab7b1f 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Sec The following provides more details on the included cryptographic software: * CloudStack makes use of JaSypt cryptographic libraries. -* CloudStack has a system requirement of MySQL, and uses native database encryption functionality. +* CloudStack requires a MySQL-compatible database (MariaDB or MySQL), and uses native database encryption functionality. * CloudStack makes use of the Bouncy Castle general-purpose encryption library. * CloudStack can optionally interact with and control OpenSwan-based VPNs. * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation. diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 0dc5b8211e0d..0a459d8d4fc5 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -78,6 +78,14 @@ zone=default # Generated with "uuidgen". local.storage.uuid= +# Enable TLS for image server transfers. The keys are read from: +# cert file = /etc/cloudstack/agent/cloud.crt +# key file = /etc/cloudstack/agent/cloud.key +image.server.tls.enabled=true + +# The Address for the network interface that the image server listens on. If not specified, it will listen on the Management network. +#image.server.listen.address= + # Location for KVM virtual router scripts. # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/". domr.scripts.dir=scripts/network/domr/kvm @@ -457,3 +465,18 @@ iscsi.session.cleanup.enabled=false # Instance conversion VIRT_V2V_TMPDIR env var #convert.instance.env.virtv2v.tmpdir= + +# Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. +# incremental.snapshot.retry.rebase.wait=60 + +# Path to the VDDK library directory for VMware to KVM conversion via VDDK, +# passed to virt-v2v as -io vddk-libdir= +#vddk.lib.dir= + +# Ordered VDDK transport preference for VMware to KVM conversion via VDDK, passed as +# -io vddk-transports= to virt-v2v. Example: nbd:nbdssl +#vddk.transports= + +# Optional vCenter SHA1 thumbprint for VMware to KVM conversion via VDDK, passed as +# -io vddk-thumbprint=. If unset, CloudStack computes it on the KVM host via openssl. +#vddk.thumbprint= diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 3364f9708cf5..18a6b6df1006 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -123,6 +123,20 @@ public class AgentProperties{ */ public static final Property LOCAL_STORAGE_PATH = new Property<>("local.storage.path", "/var/lib/libvirt/images/"); + /** + * Enables TLS on the KVM image server transfer endpoint.
+ * Data type: Boolean.
+ * Default value: true + */ + public static final Property IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", true); + + /** + * The IP address that the KVM image server listens on.
+ * Data type: String.
+ * Default value: null + */ + public static final Property IMAGE_SERVER_LISTEN_ADDRESS = new Property<>("image.server.listen.address", null, String.class); + /** * Directory where Qemu sockets are placed.
* These sockets are for the Qemu Guest Agent and SSVM provisioning.
@@ -808,6 +822,30 @@ public Property getWorkers() { */ public static final Property CONVERT_ENV_VIRTV2V_TMPDIR = new Property<>("convert.instance.env.virtv2v.tmpdir", null, String.class); + /** + * Path to the VDDK library directory on the KVM conversion host, used when converting VMs from VMware to KVM via VDDK. + * This directory is passed to virt-v2v as -io vddk-libdir=<path>. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_LIB_DIR = new Property<>("vddk.lib.dir", null, String.class); + + /** + * Ordered list of VDDK transports for virt-v2v, passed as -io vddk-transports=<value>. + * Example: nbd:nbdssl. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_TRANSPORTS = new Property<>("vddk.transports", null, String.class); + + /** + * vCenter TLS certificate thumbprint used by virt-v2v VDDK mode, passed as -io vddk-thumbprint=<value>. + * If unset, the KVM host computes it at runtime from the vCenter endpoint. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_THUMBPRINT = new Property<>("vddk.thumbprint", null, String.class); + /** * BGP controll CIDR * Data type: String.
@@ -885,6 +923,11 @@ public Property getWorkers() { */ public static final Property CREATE_FULL_CLONE = new Property<>("create.full.clone", false); + /** + * Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. + * */ + public static final Property INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60); + public static class Property { private String name; diff --git a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java index f7e4bfea80fb..fd8237998a74 100644 --- a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java @@ -26,10 +26,13 @@ public final class BucketTO { private String secretKey; + private long accountId; + public BucketTO(Bucket bucket) { this.name = bucket.getName(); this.accessKey = bucket.getAccessKey(); this.secretKey = bucket.getSecretKey(); + this.accountId = bucket.getAccountId(); } public BucketTO(String name) { @@ -47,4 +50,8 @@ public String getAccessKey() { public String getSecretKey() { return this.secretKey; } + + public long getAccountId() { + return this.accountId; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java index 18737c584b34..7daeb9649177 100644 --- a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -36,13 +36,17 @@ public class RemoteInstanceTO implements Serializable { private String vcenterPassword; private String vcenterHost; private String datacenterName; + private String clusterName; + private String hostName; public RemoteInstanceTO() { } - public RemoteInstanceTO(String instanceName) { + public RemoteInstanceTO(String instanceName, String clusterName, String hostName) { this.hypervisorType = Hypervisor.HypervisorType.VMware; this.instanceName = instanceName; + this.clusterName = clusterName; + this.hostName = hostName; } public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) { @@ -55,6 +59,12 @@ public RemoteInstanceTO(String instanceName, String instancePath, String vcenter this.datacenterName = datacenterName; } + public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName) { + this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName); + this.clusterName = clusterName; + this.hostName = hostName; + } + public Hypervisor.HypervisorType getHypervisorType() { return this.hypervisorType; } @@ -82,4 +92,12 @@ public String getVcenterHost() { public String getDatacenterName() { return datacenterName; } + + public String getClusterName() { + return clusterName; + } + + public String getHostName() { + return hostName; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index e26cc1e9f029..9af6c731fd24 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -51,6 +51,7 @@ public class VirtualMachineTO { private long minRam; private long maxRam; + private long requestedRam; private String hostName; private String arch; private String os; @@ -207,15 +208,20 @@ public long getMinRam() { return minRam; } - public void setRam(long minRam, long maxRam) { + public void setRam(long minRam, long maxRam, long requestedRam) { this.minRam = minRam; this.maxRam = maxRam; + this.requestedRam = requestedRam; } public long getMaxRam() { return maxRam; } + public long getRequestedRam() { + return requestedRam; + } + public String getHostName() { return hostName; } diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa290..5d028d31d5b6 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 8f7e773070f0..22d796d4a775 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing; + random, firstfit, userdispersing, firstfitleastconsumed; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/com/cloud/ha/Investigator.java b/api/src/main/java/com/cloud/ha/Investigator.java index 88d802a1ce44..00371d395f5a 100644 --- a/api/src/main/java/com/cloud/ha/Investigator.java +++ b/api/src/main/java/com/cloud/ha/Investigator.java @@ -26,17 +26,19 @@ public interface Investigator extends Adapter { * Returns if the vm is still alive. * * @param vm to work on. + * @return true if vm is alive, otherwise false */ - public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM; + boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM; - public Status isAgentAlive(Host agent); + /** + * Returns the agent status of the host. + * + * @param host + * @return status of the host agent + */ + Status getHostAgentStatus(Host host); class UnknownVM extends Exception { - - /** - * - */ private static final long serialVersionUID = 1L; - }; } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 9c011bac3190..b52348201516 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -57,6 +57,9 @@ public static String[] toStrings(Host.Type... types) { String HOST_UEFI_ENABLE = "host.uefi.enable"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_VDDK_SUPPORT = "host.vddk.support"; + String HOST_VDDK_LIB_DIR = "vddk.lib.dir"; + String HOST_VDDK_VERSION = "host.vddk.version"; String HOST_OVFTOOL_VERSION = "host.ovftool.version"; String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; String HOST_SSH_PORT = "host.ssh.port"; diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 0846306f70f9..e41eb880ffd5 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -510,4 +510,6 @@ public void setIp6Address(String ip6Address) { Integer getPrivateMtu(); Integer getNetworkCidrSize(); + + boolean getKeepMacAddressOnPublicNic(); } diff --git a/api/src/main/java/com/cloud/network/NetworkProfile.java b/api/src/main/java/com/cloud/network/NetworkProfile.java index 2e8efb489308..d690344a0e38 100644 --- a/api/src/main/java/com/cloud/network/NetworkProfile.java +++ b/api/src/main/java/com/cloud/network/NetworkProfile.java @@ -385,6 +385,11 @@ public Integer getNetworkCidrSize() { return networkCidrSize; } + @Override + public boolean getKeepMacAddressOnPublicNic() { + return true; + } + @Override public String toString() { return String.format("NetworkProfile %s", diff --git a/api/src/main/java/com/cloud/network/vpc/Vpc.java b/api/src/main/java/com/cloud/network/vpc/Vpc.java index b94089d2d433..a0686e2bf7d0 100644 --- a/api/src/main/java/com/cloud/network/vpc/Vpc.java +++ b/api/src/main/java/com/cloud/network/vpc/Vpc.java @@ -107,4 +107,6 @@ public enum State { String getIp6Dns2(); boolean useRouterIpAsResolver(); + + boolean getKeepMacAddressOnPublicNic(); } diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java index c1546609d2b7..3d0ba43263f5 100644 --- a/api/src/main/java/com/cloud/network/vpc/VpcService.java +++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java @@ -58,7 +58,7 @@ public interface VpcService { */ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize, - Long asNumber, List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException; + Long asNumber, List bgpPeerIds, Boolean useVrIpResolver, boolean keepMacAddressOnPublicNic) throws ResourceAllocationException; /** * Persists VPC record in the database @@ -104,7 +104,7 @@ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, Strin * @throws ResourceUnavailableException if during restart some resources may not be available * @throws InsufficientCapacityException if for instance no address space, compute or storage is sufficiently available */ - Vpc updateVpc(long vpcId, String vpcName, String displayText, String customId, Boolean displayVpc, Integer mtu, String sourceNatIp) throws ResourceUnavailableException, InsufficientCapacityException; + Vpc updateVpc(long vpcId, String vpcName, String displayText, String customId, Boolean displayVpc, Integer mtu, String sourceNatIp, Boolean keepMacAddressOnPublicNic) throws ResourceUnavailableException, InsufficientCapacityException; /** * Lists VPC(s) based on the parameters passed to the API call diff --git a/api/src/main/java/com/cloud/projects/ProjectService.java b/api/src/main/java/com/cloud/projects/ProjectService.java index 5080cb5a7812..d11e9ae0446d 100644 --- a/api/src/main/java/com/cloud/projects/ProjectService.java +++ b/api/src/main/java/com/cloud/projects/ProjectService.java @@ -82,7 +82,7 @@ public interface ProjectService { Project updateProject(long id, String name, String displayText, String newOwnerName, Long userId, Role newRole) throws ResourceAllocationException; - boolean addAccountToProject(long projectId, String accountName, String email, Long projectRoleId, Role projectRoleType); + boolean addAccountToProject(long projectId, String accountName, String email, Long projectRoleId, Role projectRoleType) throws ResourceAllocationException; boolean deleteAccountFromProject(long projectId, String accountName); @@ -100,6 +100,6 @@ public interface ProjectService { Project findByProjectAccountIdIncludingRemoved(long projectAccountId); - boolean addUserToProject(Long projectId, String username, String email, Long projectRoleId, Role projectRole); + boolean addUserToProject(Long projectId, String username, String email, Long projectRoleId, Role projectRole) throws ResourceAllocationException; } diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 51737546ffad..bcca229c06bc 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -71,7 +71,6 @@ import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; -import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.alert.Alert; import com.cloud.capacity.Capacity; @@ -108,14 +107,6 @@ public interface ManagementService { static final String Name = "management-server"; - ConfigKey JsInterpretationEnabled = new ConfigKey<>("Hidden" - , Boolean.class - , "js.interpretation.enabled" - , "false" - , "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules." - , false - , ConfigKey.Scope.Global); - /** * returns the a map of the names/values in the configuration table * @@ -534,6 +525,4 @@ VirtualMachine upgradeSystemVM(ScaleSystemVMCmd cmd) throws ResourceUnavailableE boolean removeManagementServer(RemoveManagementServerCmd cmd); - void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue); - } diff --git a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java index db702a61f2bc..7d5b2d7c57d7 100644 --- a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java +++ b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java @@ -23,9 +23,10 @@ public interface VMTemplateStorageResourceAssoc extends InternalIdentity { public static enum Status { - UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED + UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, LIMIT_REACHED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED } + List ERROR_DOWNLOAD_STATES = List.of(Status.DOWNLOAD_ERROR, Status.ABANDONED, Status.LIMIT_REACHED, Status.UNKNOWN); List PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS); String getInstallPath(); diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 1a9bcc6ee98b..43ead8d287b9 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.cloud.dc.DataCenter; import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.user.Account; @@ -70,6 +71,10 @@ public interface VolumeApiService { */ Volume allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException; + Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, String name, + Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId) + throws ResourceAllocationException; + /** * Creates the volume based on the given criteria * @@ -80,6 +85,8 @@ public interface VolumeApiService { */ Volume createVolume(CreateVolumeCmd cmd); + Volume createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display); + /** * Resizes the volume based on the given criteria * @@ -203,4 +210,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId, Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); + + Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone, boolean encryptEnabledOnly); } diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 4145e2b89eb3..fc450e9179c5 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -88,10 +88,16 @@ User createUser(String userName, String password, String firstName, String lastN Account getActiveAccountById(long accountId); + Account getActiveAccountByUuid(String accountUuid); + Account getAccount(long accountId); + Account getAccountByUuid(String accountUuid); + User getActiveUser(long userId); + User getOneActiveUserForAccount(Account account); + User getUserIncludingRemoved(long userId); boolean isRootAdmin(Long accountId); diff --git a/api/src/main/java/com/cloud/user/ResourceLimitService.java b/api/src/main/java/com/cloud/user/ResourceLimitService.java index 738e593582b4..9c493fb383c9 100644 --- a/api/src/main/java/com/cloud/user/ResourceLimitService.java +++ b/api/src/main/java/com/cloud/user/ResourceLimitService.java @@ -30,6 +30,7 @@ import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; import com.cloud.template.VirtualMachineTemplate; +import org.apache.cloudstack.resourcelimit.Reserver; public interface ResourceLimitService { @@ -191,6 +192,7 @@ public interface ResourceLimitService { */ public void checkResourceLimit(Account account, ResourceCount.ResourceType type, long... count) throws ResourceAllocationException; public void checkResourceLimitWithTag(Account account, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; + public void checkResourceLimitWithTag(Account account, Long domainId, boolean considerSystemAccount, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; /** * Gets the count of resources for a resource type and account @@ -251,12 +253,12 @@ public interface ResourceLimitService { List getResourceLimitStorageTags(DiskOffering diskOffering); void updateTaggedResourceLimitsAndCountsForAccounts(List responses, String tag); void updateTaggedResourceLimitsAndCountsForDomains(List responses, String tag); - void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; - + void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List reservations) throws ResourceAllocationException; + List getResourceLimitStorageTagsForResourceCountOperation(Boolean display, DiskOffering diskOffering); void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean display, Long currentSize, Long newSize, - DiskOffering currentOffering, DiskOffering newOffering) throws ResourceAllocationException; + DiskOffering currentOffering, DiskOffering newOffering, List reservations) throws ResourceAllocationException; - void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; + void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List reservations) throws ResourceAllocationException; void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); @@ -273,25 +275,23 @@ void updateVolumeResourceCountForDiskOfferingChange(long accountId, Boolean disp void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); - void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, List reservations) throws ResourceAllocationException; void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); void checkVmResourceLimitsForServiceOfferingChange(Account owner, Boolean display, Long currentCpu, Long newCpu, - Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template, List reservations) throws ResourceAllocationException; void checkVmResourceLimitsForTemplateChange(Account owner, Boolean display, ServiceOffering offering, - VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate) throws ResourceAllocationException; + VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate, List reservations) throws ResourceAllocationException; - void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException; void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); - void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException; void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); - void checkVmGpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu) throws ResourceAllocationException; void incrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu); void decrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu); + long recalculateDomainResourceCount(final long domainId, final ResourceType type, String tag); } diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 9e56bf4f17b2..33cc6da70812 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -130,4 +130,10 @@ public interface VmDetailConstants { String EXTERNAL_DETAIL_PREFIX = "External:"; String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details"; String CLOUDSTACK_VLAN = "cloudstack.vlan"; + + // KVM Checkpoints related + String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id"; + String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time"; + String LAST_CHECKPOINT_ID = "last.checkpoint.id"; + String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 4d33ba859a5b..e2ebb242cbf2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -127,8 +127,8 @@ public String toString() { } public static ApiCommandResourceType fromString(String value) { - if (StringUtils.isNotEmpty(value) && EnumUtils.isValidEnum(ApiCommandResourceType.class, value)) { - return valueOf(value); + if (StringUtils.isNotBlank(value) && EnumUtils.isValidEnumIgnoreCase(ApiCommandResourceType.class, value)) { + return EnumUtils.getEnumIgnoreCase(ApiCommandResourceType.class, value); } return null; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 05c6098bc726..ee10db031ba7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -69,6 +69,8 @@ public class ApiConstants { public static final String BACKUP_VM_OFFERING_REMOVED = "vmbackupofferingremoved"; public static final String IS_BACKUP_VM_EXPUNGED = "isbackupvmexpunged"; public static final String BACKUP_TOTAL = "backuptotal"; + public static final String BALANCE = "balance"; + public static final String BALANCES = "balances"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; @@ -77,6 +79,7 @@ public class ApiConstants { public static final String BOOTABLE = "bootable"; public static final String BIND_DN = "binddn"; public static final String BIND_PASSWORD = "bindpass"; + public static final String BLANK_INSTANCE = "blankinstance"; public static final String BUS_ADDRESS = "busaddress"; public static final String BYTES_READ_RATE = "bytesreadrate"; public static final String BYTES_READ_RATE_MAX = "bytesreadratemax"; @@ -157,6 +160,7 @@ public class ApiConstants { public static final String CUSTOM_ID = "customid"; public static final String CUSTOM_ACTION_ID = "customactionid"; public static final String CUSTOM_JOB_ID = "customjobid"; + public static final String CURRENCY = "currency"; public static final String CURRENT_START_IP = "currentstartip"; public static final String CURRENT_END_IP = "currentendip"; public static final String ENCRYPT = "encrypt"; @@ -170,6 +174,7 @@ public class ApiConstants { public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISKS_DETAILS = "datadisksdetails"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; + public static final String DATE = "date"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DELETE_PROTECTION = "deleteprotection"; public static final String DESCRIPTION = "description"; @@ -216,6 +221,7 @@ public class ApiConstants { public static final String DOMAIN_PATH = "domainpath"; public static final String DOMAIN_ID = "domainid"; public static final String DOMAIN__ID = "domainId"; + public static final String DUMMY = "dummy"; public static final String DURATION = "duration"; public static final String ELIGIBLE = "eligible"; public static final String EMAIL = "email"; @@ -259,6 +265,7 @@ public class ApiConstants { public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; public static final String FOR_PROVIDER = "forprovider"; + public static final String FROM_CHECKPOINT_ID = "fromcheckpointid"; public static final String FULL_PATH = "fullpath"; public static final String GATEWAY = "gateway"; public static final String IP6_GATEWAY = "ip6gateway"; @@ -331,6 +338,7 @@ public class ApiConstants { public static final String IS_2FA_VERIFIED = "is2faverified"; public static final String IS_2FA_MANDATED = "is2famandated"; + public static final String IS_ACTIVE = "isactive"; public static final String IS_ASYNC = "isasync"; public static final String IP_AVAILABLE = "ipavailable"; public static final String IP_LIMIT = "iplimit"; @@ -509,6 +517,7 @@ public class ApiConstants { public static final String REPAIR = "repair"; public static final String REPETITION_ALLOWED = "repetitionallowed"; public static final String REQUIRES_HVM = "requireshvm"; + public static final String RESERVED_RESOURCE_DETAILS = "reservedresourcedetails"; public static final String RESOURCES = "resources"; public static final String RESOURCE_COUNT = "resourcecount"; public static final String RESOURCE_NAME = "resourcename"; @@ -525,7 +534,6 @@ public class ApiConstants { public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; - public static final String USER_SECRET_KEY = "usersecretkey"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; public static final String SECURITY_GROUP_IDS = "securitygroupids"; @@ -541,6 +549,7 @@ public class ApiConstants { public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; + public static final String SHOW_RESOURCES = "showresources"; public static final String SHOW_RESOURCE_ICON = "showicon"; public static final String SHOW_INACTIVE = "showinactive"; public static final String SHOW_UNIQUE = "showunique"; @@ -606,9 +615,12 @@ public class ApiConstants { public static final String TENANT_NAME = "tenantname"; public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; + public static final String TO_CHECKPOINT_ID = "tocheckpointid"; + public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; public static final String TRUST_STORE_PASSWORD = "truststorepass"; + public static final String UNIT = "unit"; public static final String URL = "url"; public static final String USAGE_INTERFACE = "usageinterface"; public static final String USED = "used"; @@ -629,6 +641,8 @@ public class ApiConstants { public static final String USERNAME = "username"; public static final String USER_CONFIGURABLE = "userconfigurable"; public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; + public static final String USER_SECRET_KEY = "usersecretkey"; + public static final String USE_VDDK = "usevddk"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; @@ -1298,6 +1312,8 @@ public class ApiConstants { public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; public static final String QUOTA = "quota"; + public static final String QUOTA_CONSUMED = "quotaconsumed"; + public static final String QUOTA_USAGE = "quotausage"; public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; @@ -1348,6 +1364,13 @@ public class ApiConstants { public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String KEEP_MAC_ADDRESS_ON_PUBLIC_NIC = "keepmacaddressonpublicnic"; + + public static final String PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC = + "Indicates whether to use the same MAC address for the public NIC of VRs on the same network. If \"true\", when creating redundant routers or recreating" + + " a VR, CloudStack will use the same MAC address for the public NIC of all VRs. Otherwise, if \"false\", new public NICs will always have " + + " a new MAC address."; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java index 18c96c371591..1ee41ac86c22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java @@ -21,8 +21,11 @@ import javax.servlet.http.HttpSession; +import org.apache.cloudstack.context.CallContext; + import com.cloud.domain.Domain; import com.cloud.exception.CloudAuthenticationException; +import com.cloud.user.Account; import com.cloud.user.UserAccount; public interface ApiServerService { @@ -52,4 +55,20 @@ public ResponseObject loginUser(HttpSession session, String username, String pas String getDomainId(Map params); boolean isPostRequestsAndTimestampsEnforced(); + + AsyncCmdResult processAsyncCmd(BaseAsyncCmd cmdObj, Map params, CallContext ctx, Long callerUserId, Account caller) throws Exception; + + class AsyncCmdResult { + public final Long objectId; + public final String objectUuid; + public final BaseAsyncCmd asyncCmd; + public final long jobId; + + public AsyncCmdResult(Long objectId, String objectUuid, BaseAsyncCmd asyncCmd, long jobId) { + this.objectId = objectId; + this.objectUuid = objectUuid; + this.asyncCmd = asyncCmd; + this.jobId = jobId; + } + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java index 30baa71d6d85..b9b13dcfd884 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java @@ -20,7 +20,7 @@ public abstract class BaseAsyncCreateCustomIdCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.CUSTOM_ID, type = CommandType.STRING, description = "An optional field, in case you want to set a custom id to the resource. Allowed to Root Admins only") - private String customId; + protected String customId; public String getCustomId() { return customId; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index e495cf284130..00b1bc310d5a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -27,7 +27,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.regex.Pattern; import javax.inject.Inject; @@ -504,12 +503,6 @@ public Map convertExternalDetailsToMap(Map externalDetails) { } public String getResourceUuid(String parameterName) { - UUID resourceUuid = CallContext.current().getApiResourceUuid(parameterName); - - if (resourceUuid != null) { - return resourceUuid.toString(); - } - - return null; + return CallContext.current().getApiResourceUuid(parameterName); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java new file mode 100644 index 000000000000..c98bfb850529 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java @@ -0,0 +1,100 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.utils.EnumUtils; + +@APICommand(name = "createImageTransfer", + description = "Create image transfer for a disk in backup. This API is intended for testing only and is disabled by default.", + responseObject = ImageTransferResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class CreateImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + @Parameter(name = ApiConstants.VOLUME_ID, + type = CommandType.UUID, + entityType = VolumeResponse.class, + required = true, + description = "ID of the disk/volume") + private Long volumeId; + + @Parameter(name = ApiConstants.DIRECTION, + type = CommandType.STRING, + required = true, + description = "Direction of the transfer: upload, download") + private String direction; + + @Parameter(name = ApiConstants.FORMAT, + type = CommandType.STRING, + description = "Format for the image transfer: raw/cow. 'raw' will create an NBD backend. 'cow' will use the File backend." + + "For download, only the 'raw' format is supported. Default: raw") + private String format; + + public Long getBackupId() { + return backupId; + } + + public Long getVolumeId() { + return volumeId; + } + + public ImageTransfer.Direction getDirection() { + return ImageTransfer.Direction.valueOf(direction); + } + + public ImageTransfer.Format getFormat() { + return EnumUtils.getEnum(ImageTransfer.Format.class, format); + } + + @Override + public void execute() { + ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java new file mode 100644 index 000000000000..d0e17e86d427 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java @@ -0,0 +1,85 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "deleteVirtualMachineCheckpoint", + description = "Delete a VM checkpoint. This API is intended for testing only and is disabled by default.", + responseObject = SuccessResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = "checkpointid", + type = CommandType.STRING, + required = true, + description = "Checkpoint ID") + private String checkpointId; + + public Long getVmId() { + return vmId; + } + + public String getCheckpointId() { + return checkpointId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setCheckpointId(String checkpointId) { + this.checkpointId = checkpointId; + } + + @Override + public void execute() { + boolean result = kvmBackupExportService.deleteVmCheckpoint(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java new file mode 100644 index 000000000000..45173f8668ee --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeBackupCmd.java @@ -0,0 +1,103 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; + +@APICommand(name = "finalizeBackup", + description = "Finalize a VM backup session. This API is intended for testing only and is disabled by default.", + responseObject = BackupResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class FinalizeBackupCmd extends BaseAsyncCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "ID of the backup") + private Long backupId; + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + Backup backup = kvmBackupExportService.finalizeBackup(this); + + if (backup == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Finalizing backup " + backupId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java new file mode 100644 index 000000000000..dfc43e233bf2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/FinalizeImageTransferCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "finalizeImageTransfer", + description = "Finalize an image transfer. This API is intended for testing only and is disabled by default.", + responseObject = SuccessResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class FinalizeImageTransferCmd extends BaseCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + required = true, + description = "ID of the image transfer") + private Long imageTransferId; + + public Long getImageTransferId() { + return imageTransferId; + } + + @Override + public void execute() { + boolean result = kvmBackupExportService.finalizeImageTransfer(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java new file mode 100644 index 000000000000..d810d21ab5f8 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListImageTransfersCmd.java @@ -0,0 +1,81 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "listImageTransfers", + description = "List image transfers for a backup. This API is intended for testing only and is disabled by default.", + responseObject = ImageTransferResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class ListImageTransfersCmd extends BaseListCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = ImageTransferResponse.class, + description = "ID of the Image Transfer") + private Long id; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + description = "ID of the backup") + private Long backupId; + + public Long getId() { + return id; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() { + List responses = kvmBackupExportService.listImageTransfers(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java new file mode 100644 index 000000000000..a61661e982de --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/ListVmCheckpointsCmd.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.KVMBackupExportService; + +@APICommand(name = "listVirtualMachineCheckpoints", + description = "List checkpoints for a VM. This API is intended for testing only and is disabled by default.", + responseObject = CheckpointResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class ListVmCheckpointsCmd extends BaseListCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() { + List responses = kvmBackupExportService.listVmCheckpoints(this); + ListResponse response = new ListResponse<>(); + response.setResponses(responses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return 0; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java new file mode 100644 index 000000000000..1bf6d45db049 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -0,0 +1,120 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.command.admin.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.KVMBackupExportService; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; + +@APICommand(name = "startBackup", + description = "Start a VM backup session using pull mode backup-begin on the KVM host. This API is intended for testing only and is disabled by default.", + responseObject = BackupResponse.class, + since = "4.23.0", + authorized = {RoleType.Admin}) + public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { + + @Inject + private KVMBackupExportService kvmBackupExportService; + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "ID of the VM") + private Long vmId; + + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, + type = CommandType.STRING, + description = "the description for the backup") + private String description; + + public Long getVmId() { + return vmId; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + @Override + public void execute() { + try { + Backup backup = kvmBackupExportService.startBackup(this); + BackupResponse response = backupManager.createBackupResponse(backup, null); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public void create() { + Backup backup = kvmBackupExportService.createBackup(this); + + if (backup != null) { + setEntityId(backup.getId()); + setEntityUuid(backup.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Backup"); + } + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Starting backup for Instance " + vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java index 6deaea22ac6c..d333a74fdb3b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java @@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd { description = "Name of the CA service provider, otherwise the default configured provider plugin will be used") private String provider; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, + description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " + + "Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false", + since = "4.23.0") + private Boolean forced; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -79,6 +85,10 @@ public String getProvider() { return provider; } + public boolean isForced() { + return forced != null && forced; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -90,7 +100,7 @@ public void execute() { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId()); } - boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider()); + boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced()); SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index f6f66415f533..055443934609 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -19,23 +19,23 @@ import java.util.ArrayList; import java.util.List; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.commons.lang3.StringUtils; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.Pair; @@ -94,6 +94,13 @@ public class ListCfgsByCmd extends BaseListCmd { description = "The ID of the Image Store to update the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + @Parameter(name = ApiConstants.GROUP, type = CommandType.STRING, description = "Lists configuration by group name (primarily used for UI)", since = "4.18.0") private String groupName; @@ -139,6 +146,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + public String getGroupName() { return groupName; } @@ -200,6 +211,9 @@ private void setScope(ConfigurationResponse cfgResponse) { if (getImageStoreId() != null){ cfgResponse.setScope("imagestore"); } + if (getManagementServerId() != null){ + cfgResponse.setScope("managementserver"); + } } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java index 2d511cff34db..3c3c36b29d7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ResetCfgCmd.java @@ -23,16 +23,16 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.framework.config.ConfigKey; - import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.user.Account; import com.cloud.utils.Pair; @@ -84,6 +84,13 @@ public class ResetCfgCmd extends BaseCmd { description = "The ID of the Image Store to reset the parameter value for corresponding image store") private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -116,6 +123,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -149,6 +160,9 @@ public void execute() { if (getImageStoreId() != null) { response.setScope(ConfigKey.Scope.ImageStore.name()); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name()); + } response.setValue(cfg.second()); this.setResponseObject(response); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java index 97dee8f638af..9db9529dc8d8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateCfgCmd.java @@ -16,9 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.admin.config; -import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; @@ -29,13 +27,17 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConfigurationResponse; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.commons.lang3.StringUtils; import com.cloud.user.Account; +import com.cloud.utils.crypt.DBEncryptionUtil; @APICommand(name = "updateConfiguration", description = "Updates a configuration.", responseObject = ConfigurationResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -88,6 +90,13 @@ public class UpdateCfgCmd extends BaseCmd { validations = ApiArgValidator.PositiveNumber) private Long imageStoreId; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, + type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "the ID of the Management Server to update the parameter value for corresponding management server", + since = "4.23.0") + private Long managementServerId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,7 +121,7 @@ public Long getClusterId() { return clusterId; } - public Long getStoragepoolId() { + public Long getStoragePoolId() { return storagePoolId; } @@ -128,6 +137,10 @@ public Long getImageStoreId() { return imageStoreId; } + public Long getManagementServerId() { + return managementServerId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -182,7 +195,7 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getClusterId() != null) { response.setScope("cluster"); } - if (getStoragepoolId() != null) { + if (getStoragePoolId() != null) { response.setScope("storagepool"); } if (getAccountId() != null) { @@ -191,6 +204,9 @@ public ConfigurationResponse setResponseScopes(ConfigurationResponse response) { if (getDomainId() != null) { response.setScope("domain"); } + if (getManagementServerId() != null) { + response.setScope(ConfigKey.Scope.ManagementServer.name().toLowerCase()); + } return response; } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java index 6804e4355ca8..cb427e659495 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java @@ -87,6 +87,7 @@ private void setupResponse(final boolean result, final String resourceUuid) { final HostHAResponse response = new HostHAResponse(); response.setId(resourceUuid); response.setProvider(getHaProvider().toLowerCase()); + response.setStatus(result); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index abca619f82a7..4d6ef7419616 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public void execute() { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index e202dfad77ba..8f5e6c784d6e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -252,7 +252,7 @@ protected ListResponse getHostResponses() { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java index 3427cef33661..a3dcf08c3a31 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java @@ -22,7 +22,7 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.UserResponse; - +import org.apache.cloudstack.api.ApiArgValidator; import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.UserAccount; @@ -35,7 +35,7 @@ public class GetUserCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user") + @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user", validations = {ApiArgValidator.NotNullOrEmpty}) private String apiKey; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java index e11d20d06466..50ff97ac676f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmd.java @@ -85,6 +85,9 @@ public class AssignVMCmd extends BaseCmd { "In case no security groups are provided the Instance is part of the default security group.") private List securityGroupIdList; + // Internal flag to allow assignment without adding a network + private boolean skipNetwork = false; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -113,6 +116,34 @@ public List getSecurityGroupIdList() { return securityGroupIdList; } + public boolean isSkipNetwork() { + return skipNetwork; + } + + ///////////////////////////////////////////////////// + /////////////////// Setters ///////////////////////// + ///////////////////////////////////////////////////// + + public void setVirtualMachineId(Long virtualMachineId) { + this.virtualMachineId = virtualMachineId; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setSkipNetwork(boolean skipNetwork) { + this.skipNetwork = skipNetwork; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java index e64c8b3f46c6..b7c3b2ba41a6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdmin.java @@ -41,6 +41,19 @@ public class DeployVMCmdByAdmin extends DeployVMCmd implements AdminCmd { @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "Destination Cluster ID to deploy the Instance to - parameter available for root admin only", since = "4.13") private Long clusterId; + @Parameter(name = ApiConstants.BLANK_INSTANCE, + type = CommandType.BOOLEAN, + description = "Whether to create a blank instance without storage and network", + since = "4.23.0") + private Boolean blankInstance; + + // Internal flag to allow deploying instance with a given type + private String instanceType; + + ///////////////////////////////////////////////////// + ////////////////// Getters ////////////////////////// + ///////////////////////////////////////////////////// + public Long getPodId() { return podId; } @@ -48,4 +61,37 @@ public Long getPodId() { public Long getClusterId() { return clusterId; } + + @Override + public boolean isBlankInstance() { + return Boolean.TRUE.equals(blankInstance); + } + + @Override + public String getInstanceType() { + if (!isBlankInstance()) { + return null; + } + return instanceType; + } + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } + + public void setBlankInstance(boolean blankInstance) { + this.blankInstance = blankInstance; + } + + public void setInstanceType(String instanceType) { + this.instanceType = instanceType; + } + + public void setCustomId(String customId) { + this.customId = customId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java index 93d4b610b900..cbe6494d4004 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -27,4 +29,20 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance. Once destroyed, only the administrator can recover it.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd {} +public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd { + + @Parameter( name = ApiConstants.FORCED, + type = CommandType.BOOLEAN, + description = "Force destroy the Instance", + since = "4.23.0") + Boolean forced; + + @Override + public boolean isForced() { + return Boolean.TRUE.equals(forced); + } + + public void setForced(Boolean forced) { + this.forced = forced; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 50ccfbd69c5e..db7dcc3fb44f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -179,6 +179,14 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "(only for importing VMs from VMware to KVM) optional - the ID of the guest OS for the imported VM.") private Long guestOsId; + @Parameter(name = ApiConstants.USE_VDDK, + type = CommandType.BOOLEAN, + since = "4.22.1", + description = "(only for importing VMs from VMware to KVM) optional - if true, uses VDDK on the KVM conversion host for converting the VM. " + + "This parameter is mutually exclusive with " + ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES + ".") + private Boolean useVddk; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -255,6 +263,10 @@ public Long getStoragePoolId() { return storagePoolId; } + public boolean getUseVddk() { + return BooleanUtils.toBooleanDefaultIfNull(useVddk, true); + } + public String getTmpPath() { return tmpPath; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java index 85e7b0af3193..63a0a6ca51e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java @@ -18,6 +18,7 @@ import java.util.List; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.BaseCmd; @@ -106,7 +107,7 @@ public ProjectAccount.Role getRoleType() { ///////////////////////////////////////////////////// @Override - public void execute() { + public void execute() throws ResourceAllocationException { if (accountName == null && email == null) { throw new InvalidParameterValueException("Either accountName or email is required"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java index 69832d4dfdc9..683522039b17 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.account; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; @@ -111,7 +112,7 @@ public String getEventDescription() { ///////////////////////////////////////////////////// @Override - public void execute() { + public void execute() throws ResourceAllocationException { validateInput(); boolean result = _projectService.addUserToProject(getProjectId(), getUsername(), getEmail(), getProjectRoleId(), getRoleType()); if (result) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java index 28cd642e1a41..29b1e740b67e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -102,6 +103,16 @@ public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index ca60ea674fe3..8a4053aa15da 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -123,7 +123,12 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE @Override public ApiCommandResourceType getApiResourceType() { - return ApiCommandResourceType.Backup; + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return vmId; } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 67ad7c71503f..f6e17a2b3908 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; @@ -81,7 +82,7 @@ public class CreateBackupScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.QUIESCE_VM, type = CommandType.BOOLEAN, required = false, - description = "Quiesce the instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + + description = "Quiesce the Instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + "The filesystem is frozen before the backup starts and thawed immediately after. " + "Requires the instance to have the QEMU Guest Agent installed and running.", since = "4.21.0") @@ -139,4 +140,14 @@ public void execute() throws ServerApiException { public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java index dcf9f15b4dc5..62c2f03d8807 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -99,6 +100,16 @@ public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java index 703a1b2e8802..c15e6f8de684 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java @@ -20,7 +20,9 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -53,6 +55,7 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, @@ -60,12 +63,14 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { description = "ID of the Instance backup") private Long backupId; + @ACL @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.STRING, required = true, description = "ID of the volume backed up") private String volumeUuid; + @ACL @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, @@ -123,4 +128,14 @@ public String getEventType() { public String getEventDescription() { return "Restoring volume "+ volumeUuid + " from backup " + getResourceUuid(ApiConstants.BACKUP_ID) + " and attaching it to Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java index 4a9a0569c3b8..abbb1760f9d2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.bucket; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.user.Account; @@ -82,7 +83,7 @@ public ApiCommandResourceType getApiResourceType() { } @Override - public void execute() throws ConcurrentOperationException { + public void execute() throws ConcurrentOperationException, ResourceAllocationException { CallContext.current().setEventDetails("Bucket ID: " + getResourceUuid(ApiConstants.ID)); boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); SuccessResponse response = new SuccessResponse(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index b55d1b234f19..2c8401831132 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -19,6 +19,7 @@ import java.util.Date; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; @@ -40,6 +41,12 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "The id of the management server", since="4.19") private Long managementServerId; + @Parameter(name = ApiConstants.RESOURCE_ID, validations = {ApiArgValidator.UuidString}, type = CommandType.STRING, description = "the ID of the resource associated with the job", since="4.22.1") + private String resourceId; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") + private String resourceType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -52,6 +59,14 @@ public Long getManagementServerId() { return managementServerId; } + public String getResourceId() { + return resourceId; + } + + public String getResourceType() { + return resourceType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java index 93a443757212..5c3b0084574b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java @@ -16,8 +16,8 @@ // under the License. package org.apache.cloudstack.api.command.user.job; - import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; @@ -34,9 +34,15 @@ public class QueryAsyncJobResultCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, required = true, description = "The ID of the asynchronous job") + @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, description = "The ID of the asynchronous job") private Long id; + @Parameter(name = ApiConstants.RESOURCE_ID, validations = {ApiArgValidator.UuidString}, type = CommandType.STRING, description = "the ID of the resource associated with the job", since="4.22.1") + private String resourceId; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") + private String resourceType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -45,6 +51,14 @@ public Long getId() { return id; } + public String getResourceId() { + return resourceId; + } + + public String getResourceType() { + return resourceType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index cbf6df081b3b..ee5b8568e835 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -199,6 +199,11 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the network") private Long asNumber; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -286,6 +291,10 @@ public String getSourceNatIP() { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + @Override public boolean isDisplay() { if(displayNetwork == null) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java index 2e638f1e2f76..7b6841d60976 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java @@ -105,6 +105,11 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd implements UserCmd { @Parameter(name = ApiConstants.SOURCE_NAT_IP, type = CommandType.STRING, description = "IPV4 address to be assigned to the public interface of the network router. This address must already be acquired for this network", since = "4.19") private String sourceNatIP; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -186,6 +191,10 @@ public String getSourceNatIP() { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index 5c5c8776bce3..cd6baaf827bb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -193,6 +193,22 @@ public Boolean getGpuEnabled() { return gpuEnabled; } + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setCpuNumber(Integer cpuNumber) { + this.cpuNumber = cpuNumber; + } + + public void setMemory(Integer memory) { + this.memory = memory; + } + + public void setEncryptRoot(Boolean encryptRoot) { + this.encryptRoot = encryptRoot; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java index 6347c38811e8..f6ef955956f6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/AddNicToVMCmd.java @@ -100,6 +100,26 @@ public String getMacAddress() { return NetUtils.standardizeMacAddress(macaddr); } + public void setVmId(Long vmId) { + this.vmId = vmId; + } + + public void setNetworkId(Long netId) { + this.netId = netId; + } + + public void setIpaddr(String ipaddr) { + this.ipaddr = ipaddr; + } + + public void setMacAddress(String macaddr) { + this.macaddr = macaddr; + } + + public void setDhcpOptions(Map dhcpOptions) { + this.dhcpOptions = dhcpOptions; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8c29d7338b85..68b0821ba44d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -61,10 +61,10 @@ import com.cloud.network.Network.IpAddresses; import com.cloud.offering.DiskOffering; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.net.Dhcp; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; -import com.cloud.utils.net.Dhcp; public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { @@ -75,13 +75,13 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "availability zone for the virtual machine") - private Long zoneId; + protected Long zoneId; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName}) - private String name; + protected String name; @Parameter(name = ApiConstants.DISPLAY_NAME, type = CommandType.STRING, description = "an optional user generated name for the virtual machine") - private String displayName; + protected String displayName; @Parameter(name=ApiConstants.PASSWORD, type=CommandType.STRING, description="The password of the virtual machine. If null, a random password will be generated for the VM.", since="4.19.0.0") @@ -89,21 +89,21 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme //Owner information @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") - private String accountName; + protected String accountName; @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") - private Long domainId; + protected Long domainId; //Network information //@ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") - private List networkIds; + protected List networkIds; @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootType; + protected String bootType; @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootMode; + protected String bootMode; @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") private Boolean bootIntoSetup; @@ -138,7 +138,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which to deploy the virtual machine. " + "The parameter is required and respected only when hypervisor info is not set on the ISO/Template passed to the call") - private String hypervisor; + protected String hypervisor; @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + @@ -147,10 +147,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576) - private String userData; + protected String userData; @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") - private Long userdataId; + protected Long userdataId; @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18") private Map userdataDetails; @@ -160,7 +160,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String sshKeyPairName; @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs used to login to the virtual machine") - private List sshKeyPairNames; + protected List sshKeyPairNames; @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "destination Host ID to deploy the VM to - parameter available for root admin only") private Long hostId; @@ -168,7 +168,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.SECURITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups id that going to be applied to the virtual machine. " + "Should be passed only when vm is created from a zone with Basic Network support." + " Mutually exclusive with securitygroupnames parameter") - private List securityGroupIdList; + protected List securityGroupIdList; @ACL @Parameter(name = ApiConstants.SECURITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups names that going to be applied to the virtual machine." @@ -189,10 +189,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private String macAddress; @Parameter(name = ApiConstants.KEYBOARD, type = CommandType.STRING, description = "an optional keyboard device type for the virtual machine. valid value can be one of de,de-ch,es,es-latam,fi,fr,fr-be,fr-ch,is,it,jp,nl-be,no,pt,uk,us") - private String keyboard; + protected String keyboard; @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Deploy vm for the project") - private Long projectId; + protected Long projectId; @Parameter(name = ApiConstants.START_VM, type = CommandType.BOOLEAN, description = "true if start vm after creating; defaulted to true if not specified") private Boolean startVm; @@ -200,7 +200,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups id that are going to be applied to the virtual machine." + " Mutually exclusive with affinitygroupnames parameter") - private List affinityGroupIdList; + protected List affinityGroupIdList; @ACL @Parameter(name = ApiConstants.AFFINITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups names that are going to be applied to the virtual machine." @@ -208,10 +208,10 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private List affinityGroupNameList; @Parameter(name = ApiConstants.DISPLAY_VM, type = CommandType.BOOLEAN, since = "4.2", description = "an optional field, whether to the display the vm to the end user or not.", authorized = {RoleType.Admin}) - private Boolean displayVm; + protected Boolean displayVm; @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.3", description = "used to specify the custom parameters. 'extraconfig' is not allowed to be passed in details") - private Map details; + protected Map details; @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) private String deploymentPlanner; @@ -225,7 +225,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme private Map dataDiskTemplateToDiskOfferingList; @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120) - private String extraConfig; + protected String extraConfig; @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") private Boolean copyImageTags; @@ -798,6 +798,11 @@ public IoDriverPolicy getIoDriverPolicy() { } return null; } + + public String getInstanceType() { + return null; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index e17ba9c2d705..a719062e1bca 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -51,6 +51,7 @@ public class CreateVMFromBackupCmd extends BaseDeployVMCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 050592b97a3b..0a943fab118d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -16,10 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Stream; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -40,6 +41,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts an Instance based on a service offering, disk offering, and Template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, @@ -88,6 +90,111 @@ public boolean isVolumeOrSnapshotProvided() { return volumeId != null || snapshotId != null; } + public boolean isBlankInstance() { + return false; + } + + + + ///////////////////////////////////////////////////// + ////////////////// Setters ////////////////////////// + ///////////////////////////////////////////////////// + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public void setName(String name) { + this.name = name; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public void setNetworkIds(List networkIds) { + this.networkIds = networkIds; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public void setUserData(String userData) { + this.userData = userData; + } + + public void setKeyboard(String keyboard) { + this.keyboard = keyboard; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public void setDisplayVm(Boolean displayVm) { + this.displayVm = displayVm; + } + + public void setUserDataId(Long userDataId) { + this.userdataId = userDataId; + } + + public void setAffinityGroupIds(List ids) { + this.affinityGroupIdList = ids; + } + + public void setDetails(Map details) { + this.details = details; + } + + public void setExtraConfig(String extraConfig) { + this.extraConfig = extraConfig; + } + + public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { + this.dynamicScalingEnabled = dynamicScalingEnabled; + } + + public void setServiceOfferingId(Long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + public void setTemplateId(Long templateId) { + this.templateId = templateId; + } + + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public void setSshKeyPairNames(List sshKeyPairNames) { + this.sshKeyPairNames = sshKeyPairNames; + } + + public void setSecurityGroupList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + @Override public void execute() { UserVm result; @@ -132,7 +239,7 @@ public void execute() { @Override public void create() throws ResourceAllocationException { - if (Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { + if (!isBlankInstance() && Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { throw new CloudRuntimeException("Please provide only one of the following parameters - template ID, volume ID or snapshot ID"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index 9e2f2bcb72ce..aec0688f1779 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -90,6 +90,10 @@ public List getVolumeIds() { return volumeIds; } + public boolean isForced() { + return false; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index e3ad0502f454..a5f06e6ecbb3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -288,6 +288,14 @@ public boolean isCleanupExtraConfig() { return Boolean.TRUE.equals(cleanupExtraConfig); } + public void setId(Long id) { + this.id = id; + } + + public void setSecurityGroupIdList(List securityGroupIdList) { + this.securityGroupIdList = securityGroupIdList; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java index f39853512281..f50abaf73c96 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AssignVolumeCmd.java @@ -70,6 +70,21 @@ public Long getProjectid() { return projectid; } + ///////////////////////////////////////////////////// + /////////////////// Setter/////////////////////////// + ///////////////////////////////////////////////////// + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public void setProjectId(Long projectid) { + this.projectid = projectid; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 5bcf3a141178..ec7a626fa156 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -109,6 +110,14 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC description = "The ID of the Instance; to be used with snapshot Id, Instance to which the volume gets attached after creation") private Long virtualMachineId; + @Parameter(name = ApiConstants.STORAGE_ID, + type = CommandType.UUID, + entityType = StoragePoolResponse.class, + description = "Storage pool ID to create the volume in. Cannot be used with the snapshotid parameter.", + authorized = {RoleType.Admin}, + since = "4.22.1") + private Long storageId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -142,6 +151,10 @@ public Long getMaxIops() { } public Long getSnapshotId() { + if (storageId != null && snapshotId != null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Snapshot ID cannot be specified with the Storage ID."); + } return snapshotId; } @@ -153,6 +166,14 @@ private Long getProjectId() { return projectId; } + public Long getStorageId() { + if (snapshotId != null && storageId != null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Storage ID cannot be specified with the Snapshot ID."); + } + return storageId; + } + public Boolean getDisplayVolume() { return displayVolume; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java index 66a558abf982..f6e811da6055 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java @@ -77,6 +77,10 @@ public Long getVirtualMachineId() { return virtualMachineId; } + public void setId(Long id) { + this.id = id; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index 2adbbd664085..86309d7f28a5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -130,6 +130,11 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2") private Boolean useVrIpResolver; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private boolean keepMacAddressOnPublicNic = true; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -214,6 +219,10 @@ public boolean getUseVrIpResolver() { return BooleanUtils.toBoolean(useVrIpResolver); } + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java index f2327c9073f3..6cfdb895977d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java @@ -69,6 +69,11 @@ public class UpdateVPCCmd extends BaseAsyncCustomIdCmd implements UserCmd { since = "4.19") private String sourceNatIP; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -97,6 +102,10 @@ public String getSourceNatIP() { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index b855bfe40b8d..51fcaa9836e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -127,6 +127,18 @@ public class BackupResponse extends BaseResponse { @Param(description = "Indicates whether the VM from which the backup was taken is expunged or not", since = "4.22.0") private Boolean isVmExpunged; + @SerializedName(ApiConstants.FROM_CHECKPOINT_ID) + @Param(description = "Previous active checkpoint ID for incremental backups", since = "4.23.0") + private String fromCheckpointId; + + @SerializedName(ApiConstants.TO_CHECKPOINT_ID) + @Param(description = "Next checkpoint ID for incremental backups", since = "4.23.0") + private String toCheckpointId; + + @SerializedName(ApiConstants.HOST_ID) + @Param(description = "Host ID where the backup is running", since = "4.23.0") + private String hostId; + public String getId() { return id; } @@ -314,4 +326,28 @@ public void setVmOfferingRemoved(Boolean vmOfferingRemoved) { public void setVmExpunged(Boolean isVmExpunged) { this.isVmExpunged = isVmExpunged; } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + public String getFromCheckpointId() { + return this.fromCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + public String getToCheckpointId() { + return this.toCheckpointId; + } + + public void setHostId(String hostId) { + this.hostId = hostId; + } + + public String getHostId() { + return this.hostId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java new file mode 100644 index 000000000000..2bec7711064f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java @@ -0,0 +1,53 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class CheckpointResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the checkpoint ID") + private String id; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the checkpoint creation time") + private Date created; + + @SerializedName(ApiConstants.IS_ACTIVE) + @Param(description = "whether this is the active checkpoint") + private Boolean isActive; + + public void setId(String id) { + this.id = id; + } + + public void setCreated(Date created) { + this.created = created; + } + + public void setIsActive(Boolean isActive) { + this.isActive = isActive; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java index fdf1e87df506..911839da405f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java @@ -85,6 +85,12 @@ public class ExtensionResponse extends BaseResponse { @Param(description = "Removal timestamp of the extension, if applicable") private Date removed; + @SerializedName(ApiConstants.RESERVED_RESOURCE_DETAILS) + @Param(description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + public ExtensionResponse(String id, String name, String description, String type) { this.id = id; this.name = name; @@ -179,4 +185,8 @@ public Date getRemoved() { public void setRemoved(Date removed) { this.removed = removed; } + + public void setReservedResourceDetails(String reservedResourceDetails) { + this.reservedResourceDetails = reservedResourceDetails; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java new file mode 100644 index 000000000000..15576e8f1012 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageTransferResponse.java @@ -0,0 +1,104 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.backup.ImageTransfer; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = ImageTransfer.class) +public class ImageTransferResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the image transfer") + private String id; + + @SerializedName("backupid") + @Param(description = "the backup ID") + private String backupId; + + @SerializedName("vmid") + @Param(description = "the VM ID") + private String vmId; + + @SerializedName(ApiConstants.VOLUME_ID) + @Param(description = "the disk/volume ID") + private String diskId; + + @SerializedName("devicename") + @Param(description = "the device name (vda, vdb, etc)") + private String deviceName; + + @SerializedName("transferurl") + @Param(description = "the transfer URL") + private String transferUrl; + + @SerializedName("phase") + @Param(description = "the transfer phase") + private String phase; + + @SerializedName("direction") + @Param(description = "the image transfer direction: upload / download") + private String direction; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "the date created") + private Date created; + + public void setId(String id) { + this.id = id; + } + + public void setBackupId(String backupId) { + this.backupId = backupId; + } + + public void setVmId(String vmId) { + this.vmId = vmId; + } + + public void setDiskId(String diskId) { + this.diskId = diskId; + } + + public void setDeviceName(String deviceName) { + this.deviceName = deviceName; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java index 7c4b733a80f5..3a3663af2551 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java @@ -331,6 +331,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement @Param(description = "The BGP peers for the network", since = "4.20.0") private Set bgpPeers; + @SerializedName(ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, since = "4.23.0") + private Boolean keepMacAddressOnPublicNic; + public NetworkResponse() {} public Boolean getDisplayNetwork() { @@ -702,4 +706,8 @@ public void setIpv6Dns1(String ipv6Dns1) { public void setIpv6Dns2(String ipv6Dns2) { this.ipv6Dns2 = ipv6Dns2; } + + public void setKeepMacAddressOnPublicNic(Boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java index acfabb113502..34d50d5b9f92 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java @@ -185,6 +185,10 @@ public class VpcResponse extends BaseResponseWithAnnotations implements Controll @Param(description = "The BGP peers for the VPC", since = "4.20.0") private Set bgpPeers; + @SerializedName(ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, since = "4.23.0") + private Boolean keepMacAddressOnPublicNic; + public void setId(final String id) { this.id = id; } @@ -366,4 +370,8 @@ public void addBgpPeer(BgpPeerResponse bgpPeer) { } this.bgpPeers.add(bgpPeer); } + + public void setKeepMacAddressOnPublicNic(Boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 951af9180e7f..2d68f18b953f 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -30,8 +30,16 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { + String getFromCheckpointId(); + + String getToCheckpointId(); + + Long getCheckpointCreateTime(); + + Long getHostId(); + enum Status { - Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged + Allocated, Queued, BackingUp, ReadyForImageTransfer, FinalizingImageTransfer, BackedUp, Error, Failed, Restoring, Removed, Expunged } class Metric { diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 6c0121a3e4d8..0da11bbd651a 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -235,7 +235,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer * @param forced Indicates if backup will be force removed or not * @return returns operation success */ - boolean deleteBackup(final Long backupId, final Boolean forced); + boolean deleteBackup(final Long backupId, final Boolean forced) throws ResourceAllocationException; void validateBackupForZone(Long zoneId); diff --git a/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java new file mode 100644 index 000000000000..e1153be3ae02 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/ImageTransfer.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ImageTransfer extends ControlledEntity, InternalIdentity { + long getDataCenterId(); + + public enum Direction { + upload, download + } + + public enum Format { + raw, + cow + } + + public enum Backend { + nbd, + file + } + + public enum Phase { + initializing, transferring, finished, failed + } + + String getUuid(); + + Long getBackupId(); + + long getVolumeId(); + + long getHostId(); + + String getTransferUrl(); + + Phase getPhase(); + + Direction getDirection(); + + Backend getBackend(); + + String getSignedTicketId(); +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java new file mode 100644 index 000000000000..e1affb311ec6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -0,0 +1,106 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.List; + +import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.DeleteVmCheckpointCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeBackupCmd; +import org.apache.cloudstack.api.command.admin.backup.FinalizeImageTransferCmd; +import org.apache.cloudstack.api.command.admin.backup.ListImageTransfersCmd; +import org.apache.cloudstack.api.command.admin.backup.ListVmCheckpointsCmd; +import org.apache.cloudstack.api.command.admin.backup.StartBackupCmd; +import org.apache.cloudstack.api.response.CheckpointResponse; +import org.apache.cloudstack.api.response.ImageTransferResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.utils.component.PluggableService; + +/** + * Service for Creating Backups and ImageTransfer sessions which will be consumed by an external orchestrator. + */ +public interface KVMBackupExportService extends Configurable, PluggableService { + + ConfigKey ImageTransferIdleTimeoutSeconds = new ConfigKey<>("Advanced", Integer.class, + "image.transfer.idle.timeout.seconds", + "600", + "Seconds since last completed HTTP request to an image transfer before the image server unregisters it (idle timeout).", + true, ConfigKey.Scope.Zone); + + ConfigKey ExposeKVMBackupExportServiceApis = new ConfigKey<>("Advanced", Boolean.class, + "expose.kvm.backup.export.service.apis", + "false", + "Enable to expose APIs for testing the KVM Backup Export Service.", + false, ConfigKey.Scope.Global); + /** + * Creates a backup session for a VM + */ + Backup createBackup(StartBackupCmd cmd); + + /** + * Start a backup session for a VM + * Creates a new checkpoint and starts NBD server for pull-mode backup + */ + Backup startBackup(StartBackupCmd cmd); + + /** + * Finalize a backup session + * Stops NBD server, updates checkpoint tracking, deletes old checkpoints + */ + Backup finalizeBackup(FinalizeBackupCmd cmd); + + /** + * Create an image transfer object for a disk + * Registers NBD endpoint with ImageIO (stubbed for POC) + */ + ImageTransferResponse createImageTransfer(CreateImageTransferCmd cmd); + + ImageTransfer createImageTransfer(long volumeId, Long backupId, ImageTransfer.Direction direction, ImageTransfer.Format format); + + boolean cancelImageTransfer(long imageTransferId); + + /** + * Finalize an image transfer + * Marks transfer as complete (NBD is closed globally in finalize backup) + */ + boolean finalizeImageTransfer(FinalizeImageTransferCmd cmd); + + boolean finalizeImageTransfer(long imageTransferId); + + /** + * List image transfers for a backup + */ + List listImageTransfers(ListImageTransfersCmd cmd); + + /** + * List checkpoints for a VM + */ + List listVmCheckpoints(ListVmCheckpointsCmd cmd); + + /** + * Delete a VM checkpoint (no-op for normal flow, kept for API parity) + */ + boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd); + + /** + * List Compatible Data Center Ids for the service + */ + List listCompatibleDataCenterIds(); +} diff --git a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java index b0fb1ac73c21..d2ebdc25f1bd 100644 --- a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java +++ b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; +import com.trilead.ssh2.Connection; + import org.apache.cloudstack.framework.ca.CAProvider; import org.apache.cloudstack.framework.ca.CAService; import org.apache.cloudstack.framework.ca.Certificate; @@ -39,7 +41,10 @@ public interface CAManager extends CAService, Configurable, PluggableService { ConfigKey CAProviderPlugin = new ConfigKey<>("Advanced", String.class, "ca.framework.provider.plugin", "root", - "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true); + "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " + + "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " + + "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " + + "Restart management server(s) when changed.", false); ConfigKey CertKeySize = new ConfigKey<>("Advanced", Integer.class, "ca.framework.cert.keysize", @@ -85,6 +90,12 @@ public interface CAManager extends CAService, Configurable, PluggableService { "The actual implementation will depend on the configured CA provider.", false); + ConfigKey CaInjectDefaultTruststore = new ConfigKey<>("Advanced", Boolean.class, + "ca.framework.inject.default.truststore", "true", + "When true, injects the CA provider's certificate into the JVM default truststore on management server startup. " + + "This allows outgoing HTTPS connections from the management server to trust servers with certificates signed by the configured CA. " + + "Restart management server(s) when changed.", false); + /** * Returns a list of available CA provider plugins * @return returns list of CAProvider @@ -130,12 +141,26 @@ public interface CAManager extends CAService, Configurable, PluggableService { boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider); /** - * Provisions certificate for given active and connected agent host + * Provisions certificate for given agent host. + * When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents). * @param host + * @param reconnect * @param provider + * @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs * @return returns success/failure as boolean */ - boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider); + boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced); + + /** + * Provisions certificate for a KVM host using an existing SSH connection. + * Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import. + * Used during host discovery and for forced re-provisioning when the NIO agent is unreachable. + * @param sshConnection active SSH connection to the KVM host + * @param agentIp IP address of the KVM host agent + * @param agentHostname hostname of the KVM host agent + * @param caProvider optional CA provider plugin name (null uses default) + */ + void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname, String caProvider); /** * Setups up a new keystore and generates CSR for a host diff --git a/api/src/main/java/org/apache/cloudstack/context/CallContext.java b/api/src/main/java/org/apache/cloudstack/context/CallContext.java index 5e0c60184f4f..fcfb5b6b1e00 100644 --- a/api/src/main/java/org/apache/cloudstack/context/CallContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/CallContext.java @@ -63,7 +63,7 @@ protected Stack initialValue() { private User user; private long userId; private final Map context = new HashMap(); - private final Map apiResourcesUuids = new HashMap<>(); + private final Map apiResourcesUuids = new HashMap<>(); private Project project; private String apiName; @@ -389,11 +389,11 @@ public void setEventDisplayEnabled(boolean eventDisplayEnabled) { isEventDisplayEnabled = eventDisplayEnabled; } - public UUID getApiResourceUuid(String paramName) { + public String getApiResourceUuid(String paramName) { return apiResourcesUuids.get(paramName); } - public void putApiResourceUuid(String paramName, UUID uuid) { + public void putApiResourceUuid(String paramName, String uuid) { apiResourcesUuids.put(paramName, uuid); } diff --git a/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java b/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java index f50f841ed74a..a01131278a76 100644 --- a/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java +++ b/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java @@ -17,8 +17,11 @@ package org.apache.cloudstack.extension; +import java.util.List; + public interface ExtensionHelper { Long getExtensionIdForCluster(long clusterId); Extension getExtension(long id); Extension getExtensionForCluster(long clusterId); + List getExtensionReservedResourceDetails(long extensionId); } diff --git a/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java b/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java new file mode 100644 index 000000000000..6b3f57b6aa5e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.resourcelimit; + +/** + * Interface implemented by CheckedReservation. + *

+ * This is defined in cloud-api to allow methods declared in modules that do not depend on cloud-server + * to receive CheckedReservations as parameters. + */ +public interface Reserver extends AutoCloseable { + + void close(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index e27ef308d7f2..8c164133db88 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -95,7 +95,7 @@ public interface BucketApiService { */ Bucket createBucket(CreateBucketCmd cmd); - boolean deleteBucket(long bucketId, Account caller); + boolean deleteBucket(long bucketId, Account caller) throws ResourceAllocationException; boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java b/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java index 21184de27a26..3e349cf0bd39 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/sharedfs/SharedFSService.java @@ -23,6 +23,10 @@ import org.apache.cloudstack.api.command.user.storage.sharedfs.ChangeSharedFSServiceOfferingCmd; import org.apache.cloudstack.api.command.user.storage.sharedfs.CreateSharedFSCmd; import org.apache.cloudstack.api.command.user.storage.sharedfs.DestroySharedFSCmd; +import org.apache.cloudstack.api.command.user.storage.sharedfs.ListSharedFSCmd; +import org.apache.cloudstack.api.command.user.storage.sharedfs.UpdateSharedFSCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SharedFSResponse; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; @@ -31,11 +35,6 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.VirtualMachineMigrationException; -import org.apache.cloudstack.api.command.user.storage.sharedfs.ListSharedFSCmd; -import org.apache.cloudstack.api.command.user.storage.sharedfs.UpdateSharedFSCmd; -import org.apache.cloudstack.api.response.SharedFSResponse; -import org.apache.cloudstack.api.response.ListResponse; - public interface SharedFSService { List getSharedFSProviders(); @@ -69,4 +68,10 @@ public interface SharedFSService { SharedFS recoverSharedFS(Long sharedFSId); void deleteSharedFS(Long sharedFSId); + + SharedFS getSharedFSByUuid(String uuid); + + SharedFS getSharedFSForVmId(long vmId); + + SharedFS updateSharedFSPostRestore(long sharedFsId, long volumeId); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java index 78c598272e03..13f01b4944ad 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -37,7 +37,7 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware); List SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD); + Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint); ConfigKey AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class, "allow.import.volume.with.backing.file", diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java index 4039ca6dc948..38d0df2e8b82 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java @@ -40,7 +40,7 @@ public class CreateIpv4SubnetForGuestNetworkCmdTest { @Test public void testCreateIpv4SubnetForGuestNetworkCmd() { Long parentId = 1L; - UUID parentUuid = UUID.randomUUID(); + String parentUuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; Integer cidrSize = 26; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java index bb324aca0e70..560ecdc3b296 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java @@ -40,7 +40,7 @@ public class CreateIpv4SubnetForZoneCmdTest { @Test public void testCreateIpv4SubnetForZoneCmd() { Long zoneId = 1L; - UUID zoneUuid = UUID.randomUUID(); + String zoneUuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; String accountName = "user"; Long projectId = 10L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java index 31458d2833fa..4640510ccdab 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class DedicateIpv4SubnetForZoneCmdTest { @Test public void testDedicateIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java index 48aceeaaeec1..cd25d8d24014 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java @@ -39,7 +39,7 @@ public class DeleteIpv4SubnetForGuestNetworkCmdTest { @Test public void testDeleteIpv4SubnetForGuestNetworkCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteIpv4SubnetForGuestNetworkCmd cmd = new DeleteIpv4SubnetForGuestNetworkCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java index 5c3593a8f1b6..269fb3f3c192 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class DeleteIpv4SubnetForZoneCmdTest { @Test public void testDeleteIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteIpv4SubnetForZoneCmd cmd = new DeleteIpv4SubnetForZoneCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java index 29d6d8e735bb..6e5d2a95f3ab 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class ReleaseDedicatedIpv4SubnetForZoneCmdTest { @Test public void testReleaseDedicatedIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReleaseDedicatedIpv4SubnetForZoneCmd cmd = new ReleaseDedicatedIpv4SubnetForZoneCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java index 399b77de6e85..af37006eafd2 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java @@ -40,7 +40,7 @@ public class UpdateIpv4SubnetForZoneCmdTest { @Test public void testUpdateIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; UpdateIpv4SubnetForZoneCmd cmd = new UpdateIpv4SubnetForZoneCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java index 9cd403ddd1de..3db1fab466f6 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java @@ -46,7 +46,7 @@ public class ChangeBgpPeersForNetworkCmdTest { @Test public void testChangeBgpPeersForNetworkCmd() { Long networkId = 10L; - UUID networkUuid = UUID.randomUUID(); + String networkUuid = UUID.randomUUID().toString(); List bgpPeerIds = Arrays.asList(20L, 21L); ChangeBgpPeersForNetworkCmd cmd = new ChangeBgpPeersForNetworkCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java index 545523e3ab97..fb85f7060684 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java @@ -46,7 +46,7 @@ public class ChangeBgpPeersForVpcCmdTest { @Test public void testChangeBgpPeersForVpcCmd() { Long VpcId = 10L; - UUID vpcUuid = UUID.randomUUID(); + String vpcUuid = UUID.randomUUID().toString(); List bgpPeerIds = Arrays.asList(20L, 21L); ChangeBgpPeersForVpcCmd cmd = new ChangeBgpPeersForVpcCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java index 866824f62936..3abc5f57d017 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java @@ -40,7 +40,7 @@ public class CreateBgpPeerCmdTest { @Test public void testCreateBgpPeerCmd() { Long zoneId = 1L; - UUID zoneUuid = UUID.randomUUID(); + String zoneUuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java index a8046d3d745c..c5edb1b8f53f 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class DedicateBgpPeerCmdTest { @Test public void testDedicateBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java index d54be9e859e4..5228a63dc92d 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class DeleteBgpPeerCmdTest { @Test public void testDeleteBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteBgpPeerCmd cmd = new DeleteBgpPeerCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java index 1cf8ead706d1..60a814d63050 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class ReleaseDedicatedBgpPeerCmdTest { @Test public void testReleaseDedicatedBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReleaseDedicatedBgpPeerCmd cmd = new ReleaseDedicatedBgpPeerCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java index 1601fcb4c5a5..d594bc5718b7 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java @@ -40,7 +40,7 @@ public class UpdateBgpPeerCmdTest { @Test public void testUpdateBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String ip4Address = "ip4-address"; String ip6Address = "ip6-address"; Long peerAsNumber = 15000L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java index 3b2b49cae4e1..8d2771c969b7 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java @@ -97,7 +97,7 @@ public void testGetEventType() { @Test public void testGetEventDescription() { - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReflectionTestUtils.setField(cmd, "storeId", 1L); ReflectionTestUtils.setField(cmd, "path", "path/to/object"); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java new file mode 100644 index 000000000000..27bc4614e1b5 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/AssignVMCmdTest.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class AssignVMCmdTest { + + @Test + public void test_setSkipNetwork_default() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + Object value = ReflectionTestUtils.getField(assignVMCmd, "skipNetwork"); + Assert.assertTrue(value instanceof Boolean); + Assert.assertFalse((Boolean) value); + } + + @Test + public void test_setSkipNetwork_set() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + assignVMCmd.setSkipNetwork(true); + Object value = ReflectionTestUtils.getField(assignVMCmd, "skipNetwork"); + Assert.assertTrue(value instanceof Boolean); + Assert.assertTrue((Boolean) value); + } + + @Test + public void test_isSkipNetwork_default() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + Assert.assertFalse(assignVMCmd.isSkipNetwork()); + } + + @Test + public void test_isSkipNetwork_set() { + AssignVMCmd assignVMCmd = new AssignVMCmd(); + ReflectionTestUtils.setField(assignVMCmd, "skipNetwork", true); + Assert.assertTrue(assignVMCmd.isSkipNetwork()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java new file mode 100644 index 000000000000..d82dc9f766f9 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/DeployVMCmdByAdminTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class DeployVMCmdByAdminTest { + + @InjectMocks + private DeployVMCmdByAdmin cmd; + + @Test + public void testIsBlankInstance_default() { + assertFalse(cmd.isBlankInstance()); + } + + @Test + public void testIsBlankInstance_true() { + ReflectionTestUtils.setField(cmd, "blankInstance", true); + assertTrue(cmd.isBlankInstance()); + } + + @Test + public void testIsBlankInstance_false() { + ReflectionTestUtils.setField(cmd, "blankInstance", false); + assertFalse(cmd.isBlankInstance()); + } + + @Test + public void testSetBlankInstance_default() { + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNull(obj); + } + + @Test + public void testSetBlankInstance_true() { + cmd.setBlankInstance(true); + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNotNull(obj); + assertTrue((boolean)obj); + } + + @Test + public void testSetBlankInstance_false() { + cmd.setBlankInstance(false); + Object obj = ReflectionTestUtils.getField(cmd, "blankInstance"); + assertNotNull(obj); + assertFalse((boolean)obj); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java index ecca507a6b95..59a61806e867 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java @@ -44,7 +44,7 @@ public class UnmanageVolumeCmdTest { public void testUnmanageVolumeCmd() { long accountId = 2L; Long volumeId = 3L; - UUID volumeUuid = UUID.randomUUID(); + String volumeUuid = UUID.randomUUID().toString(); Volume volume = Mockito.mock(Volume.class); Mockito.when(responseGenerator.findVolumeById(volumeId)).thenReturn(volume); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java index f100822b8c77..cd0390aa2689 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.test; +import com.cloud.exception.ResourceAllocationException; import junit.framework.Assert; import junit.framework.TestCase; @@ -149,6 +150,8 @@ public void testExecuteForNullAccountNameEmail() { addAccountToProjectCmd.execute(); } catch (InvalidParameterValueException exception) { Assert.assertEquals("Either accountName or email is required", exception.getLocalizedMessage()); + } catch (ResourceAllocationException exception) { + Assert.fail(); } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index b70efaf9a6c5..032dca8d8007 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -118,7 +118,7 @@ public void testCreateFailure() { AccountService accountService = Mockito.mock(AccountService.class); Account account = Mockito.mock(Account.class); Mockito.when(accountService.getAccount(anyLong())).thenReturn(account); - UUID volumeUuid = UUID.randomUUID(); + String volumeUuid = UUID.randomUUID().toString(); CallContext.current().putApiResourceUuid("volumeid", volumeUuid); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java index 3dfb29dadd3e..c78dbe9b56bc 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java @@ -56,7 +56,7 @@ public class UpdateConditionCmdTest { private static final Long threshold = 100L; private static final long accountId = 5L; - private static final UUID conditionUuid = UUID.randomUUID(); + private static final String conditionUuid = UUID.randomUUID().toString(); @Before public void setUp() { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java index d3cf5dd6cd68..dbe7669431d8 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java @@ -49,7 +49,7 @@ public void testProperties() { ReflectionTestUtils.setField(cmd, "_firewallService", _firewallService); long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); long accountId = 2L; long networkId = 3L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java index f7e3e38d9c3f..09d396e40237 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.vm; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -41,6 +42,7 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.NetworkService; import com.cloud.utils.db.EntityManager; import com.cloud.vm.VmDetailConstants; @@ -480,4 +482,146 @@ public void testGetDataDiskTemplateToDiskOfferingMapInvalidTemplateId() { }); assertTrue(thrownException.getMessage().contains("Unable to translate and find entity with datadisktemplateid")); } + + @Test + public void testSetServiceOfferingId() { + cmd.setServiceOfferingId(101L); + assertEquals(Long.valueOf(101L), cmd.getServiceOfferingId()); + } + + @Test + public void testSetTemplateId() { + cmd.setTemplateId(102L); + assertEquals(Long.valueOf(102L), cmd.getTemplateId()); + } + + @Test + public void testSetVolumeId() { + cmd.setVolumeId(103L); + assertEquals(Long.valueOf(103L), cmd.getVolumeId()); + } + + @Test + public void testSetSnapshotId() { + cmd.setSnapshotId(104L); + assertEquals(Long.valueOf(104L), cmd.getSnapshotId()); + } + + @Test + public void testSetZoneId() { + cmd.setZoneId(105L); + assertEquals(Long.valueOf(105L), cmd.getZoneId()); + } + + @Test + public void testSetName() { + cmd.setName("vm-name"); + assertEquals("vm-name", cmd.getName()); + } + + @Test + public void testSetDisplayName() { + cmd.setDisplayName("vm-display-name"); + assertEquals("vm-display-name", cmd.getDisplayName()); + } + + @Test + public void testSetAccountName() { + cmd.setAccountName("account-name"); + assertEquals("account-name", cmd.getAccountName()); + } + + @Test + public void testSetDomainId() { + cmd.setDomainId(106L); + assertEquals(Long.valueOf(106L), cmd.getDomainId()); + } + + @Test + public void testSetNetworkIds() { + List networkIds = Arrays.asList(11L, 12L); + cmd.setNetworkIds(networkIds); + assertEquals(networkIds, cmd.getNetworkIds()); + } + + @Test + public void testSetBootType() { + cmd.setBootType("UEFI"); + assertEquals(BootType.UEFI, cmd.getBootType()); + } + + @Test + public void testSetBootMode() { + cmd.setBootType("UEFI"); + cmd.setBootMode("SECURE"); + assertEquals(BootMode.SECURE, cmd.getBootMode()); + } + + @Test + public void testSetHypervisor() { + cmd.setHypervisor("KVM"); + assertEquals(HypervisorType.KVM, cmd.getHypervisor()); + } + + @Test + public void testSetUserData() { + cmd.setUserData("dXNlci1kYXRh"); + assertEquals("dXNlci1kYXRh", cmd.getUserData()); + } + + @Test + public void testSetKeyboard() { + cmd.setKeyboard("us"); + assertEquals("us", cmd.getKeyboard()); + } + + @Test + public void testSetProjectId() { + cmd.setProjectId(107L); + assertEquals(Long.valueOf(107L), ReflectionTestUtils.getField(cmd, "projectId")); + } + + @Test + public void testSetDisplayVm() { + cmd.setDisplayVm(Boolean.FALSE); + assertEquals(Boolean.FALSE, cmd.isDisplayVm()); + } + + @Test + public void testSetUserDataId() { + cmd.setUserDataId(108L); + assertEquals(Long.valueOf(108L), cmd.getUserdataId()); + } + + @Test + public void testSetAffinityGroupIds() { + List affinityGroupIds = Arrays.asList(21L, 22L); + cmd.setAffinityGroupIds(affinityGroupIds); + assertEquals(affinityGroupIds, cmd.getAffinityGroupIdList()); + } + + @Test + public void testSetDetails() { + Map details = new HashMap<>(); + details.put("key", "value"); + cmd.setDetails(details); + assertEquals(details, ReflectionTestUtils.getField(cmd, "details")); + } + + @Test + public void testSetExtraConfig() { + cmd.setExtraConfig("cpu-mode=host-passthrough"); + assertEquals("cpu-mode=host-passthrough", cmd.getExtraConfig()); + } + + @Test + public void testSetDynamicScalingEnabled() { + cmd.setDynamicScalingEnabled(Boolean.FALSE); + assertFalse(cmd.isDynamicScalingEnabled()); + } + + @Test + public void testIsBlankInstance() { + assertFalse(cmd.isBlankInstance()); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java index acb2dc685976..9d56cffd6718 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java @@ -93,7 +93,7 @@ public void testExecute() throws ResourceUnavailableException, InsufficientCapac responseGenerator = Mockito.mock(ResponseGenerator.class); cmd._responseGenerator = responseGenerator; Mockito.verify(_vpcService, Mockito.times(0)).updateVpc(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyBoolean()); } } diff --git a/client/pom.xml b/client/pom.xml index b8dffe65d4fb..7d37225dbbb3 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -121,6 +121,11 @@ cloud-plugin-storage-volume-adaptive ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-volume-ontap + ${project.version} + org.apache.cloudstack cloud-plugin-storage-volume-solidfire @@ -372,11 +377,6 @@ cloud-plugin-explicit-dedication ${project.version} - - org.apache.cloudstack - cloud-plugin-host-allocator-random - ${project.version} - org.apache.cloudstack cloud-plugin-outofbandmanagement-driver-ipmitool @@ -612,6 +612,11 @@ cloud-plugin-backup-nas ${project.version} + + org.apache.cloudstack + cloud-plugin-integrations-veeam-control-service + ${project.version} + org.apache.cloudstack cloud-plugin-integrations-kubernetes-service @@ -716,17 +721,17 @@ org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on ${cs.bcprov.version} org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${cs.bcprov.version} org.bouncycastle - bctls-jdk15on + bctls-jdk18on ${cs.bcprov.version} @@ -906,13 +911,13 @@ org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on false ${project.build.directory}/lib org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on false ${project.build.directory}/lib @@ -936,7 +941,7 @@ org.bouncycastle - bctls-jdk15on + bctls-jdk18on false ${project.build.directory}/lib @@ -971,9 +976,9 @@ org.apache.tomcat.embed:tomcat-embed-core org.apache.geronimo.specs:geronimo-servlet_3.0_spec org.apache.geronimo.specs:geronimo-javamail_1.4_spec - org.bouncycastle:bcprov-jdk15on - org.bouncycastle:bcpkix-jdk15on - org.bouncycastle:bctls-jdk15on + org.bouncycastle:bcprov-jdk18on + org.bouncycastle:bcpkix-jdk18on + org.bouncycastle:bctls-jdk18on com.mysql:mysql-connector-j org.apache.cloudstack:cloud-plugin-storage-volume-storpool org.apache.cloudstack:cloud-plugin-storage-volume-linstor diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java index fc066e5c5899..c46fb697a3c1 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java @@ -18,6 +18,8 @@ public class CheckConvertInstanceCommand extends Command { boolean checkWindowsGuestConversionSupport = false; + boolean useVddk = false; + String vddkLibDir; public CheckConvertInstanceCommand() { } @@ -26,6 +28,11 @@ public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport) { this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; } + public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport, boolean useVddk) { + this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; + this.useVddk = useVddk; + } + @Override public boolean executeInSequence() { return false; @@ -34,4 +41,20 @@ public boolean executeInSequence() { public boolean getCheckWindowsGuestConversionSupport() { return checkWindowsGuestConversionSupport; } + + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } } diff --git a/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java index 5a26b22ec6a5..41e266784db8 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java @@ -38,6 +38,8 @@ public CheckOnHostAnswer(CheckOnHostCommand cmd, Boolean alive, String details) public CheckOnHostAnswer(CheckOnHostCommand cmd, String details) { super(cmd, false, details); + determined = false; + alive = false; } public boolean isDetermined() { @@ -47,5 +49,4 @@ public boolean isDetermined() { public boolean isAlive() { return alive; } - } diff --git a/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java b/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java index 94239f2900ef..72b5217604dc 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java @@ -24,7 +24,7 @@ public class CheckOnHostCommand extends Command { HostTO host; - boolean reportCheckFailureIfOneStorageIsDown; + boolean reportIfHeartBeatFailedForOneStoragePool; protected CheckOnHostCommand() { } @@ -34,17 +34,17 @@ public CheckOnHostCommand(Host host) { setWait(20); } - public CheckOnHostCommand(Host host, boolean reportCheckFailureIfOneStorageIsDown) { + public CheckOnHostCommand(Host host, boolean reportIfHeartBeatFailedForOneStoragePool) { this(host); - this.reportCheckFailureIfOneStorageIsDown = reportCheckFailureIfOneStorageIsDown; + this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool; } public HostTO getHost() { return host; } - public boolean isCheckFailedOnOneStorage() { - return reportCheckFailureIfOneStorageIsDown; + public boolean shouldReportIfHeartBeatFailedForOneStoragePool() { + return reportIfHeartBeatFailedForOneStoragePool; } @Override diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index 24336747ccf4..38e0dca7736c 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -31,6 +31,10 @@ public class ConvertInstanceCommand extends Command { private boolean exportOvfToConversionLocation; private int threadsCountToExportOvf = 0; private String extraParams; + private boolean useVddk; + private String vddkLibDir; + private String vddkTransports; + private String vddkThumbprint; public ConvertInstanceCommand() { } @@ -90,6 +94,38 @@ public void setExtraParams(String extraParams) { this.extraParams = extraParams; } + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public void setVddkTransports(String vddkTransports) { + this.vddkTransports = vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public void setVddkThumbprint(String vddkThumbprint) { + this.vddkThumbprint = vddkThumbprint; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java index ed337885beed..21c4e7b97d03 100644 --- a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java +++ b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java @@ -24,6 +24,8 @@ public class PropagateResourceEventCommand extends Command { long hostId; ResourceState.Event event; + boolean forced; + boolean forceDeleteStorage; protected PropagateResourceEventCommand() { @@ -34,6 +36,13 @@ public PropagateResourceEventCommand(long hostId, ResourceState.Event event) { this.event = event; } + public PropagateResourceEventCommand(long hostId, ResourceState.Event event, boolean forced, boolean forceDeleteStorage) { + this.hostId = hostId; + this.event = event; + this.forced = forced; + this.forceDeleteStorage = forceDeleteStorage; + } + public long getHostId() { return hostId; } @@ -42,6 +51,14 @@ public ResourceState.Event getEvent() { return event; } + public boolean isForced() { + return forced; + } + + public boolean isForceDeleteStorage() { + return forceDeleteStorage; + } + @Override public boolean executeInSequence() { // TODO Auto-generated method stub diff --git a/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java b/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java index dc63b1ee746d..e2953e9c7ad7 100644 --- a/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java @@ -30,6 +30,7 @@ public class ScaleVmCommand extends Command { Integer maxSpeed; long minRam; long maxRam; + private boolean limitCpuUseChange; public VirtualMachineTO getVm() { return vm; @@ -43,7 +44,7 @@ public int getCpus() { return cpus; } - public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse) { + public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse, Double cpuQuotaPercentage, boolean limitCpuUseChange) { super(); this.vmName = vmName; this.cpus = cpus; @@ -52,6 +53,8 @@ public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpee this.minRam = minRam; this.maxRam = maxRam; this.vm = new VirtualMachineTO(1L, vmName, null, cpus, minSpeed, maxSpeed, minRam, maxRam, null, null, false, limitCpuUse, null); + this.vm.setCpuQuotaPercentage(cpuQuotaPercentage); + this.limitCpuUseChange = limitCpuUseChange; } public void setCpus(int cpus) { @@ -102,6 +105,10 @@ public VirtualMachineTO getVirtualMachine() { return vm; } + public boolean getLimitCpuUseChange() { + return limitCpuUseChange; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java index d8cc74817d7b..96d73e11990d 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java @@ -36,6 +36,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { public String lbStatsAuth = "admin1:AdMiN123"; public String lbStatsUri = "/admin?stats"; public String maxconn = ""; + public Long idleTimeout = 50000L; /* 0=infinite, >0 = timeout in milliseconds */ public String lbProtocol; public boolean keepAliveEnabled = false; NicTO nic; @@ -50,7 +51,7 @@ public LoadBalancerConfigCommand(LoadBalancerTO[] loadBalancers, Long vpcId) { } public LoadBalancerConfigCommand(LoadBalancerTO[] loadBalancers, String publicIp, String guestIp, String privateIp, NicTO nic, Long vpcId, String maxconn, - boolean keepAliveEnabled) { + boolean keepAliveEnabled, Long idleTimeout) { this.loadBalancers = loadBalancers; this.lbStatsPublicIP = publicIp; this.lbStatsPrivateIP = privateIp; @@ -59,6 +60,7 @@ public LoadBalancerConfigCommand(LoadBalancerTO[] loadBalancers, String publicIp this.vpcId = vpcId; this.maxconn = maxconn; this.keepAliveEnabled = keepAliveEnabled; + this.idleTimeout = idleTimeout; } public NicTO getNic() { diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 0c6373134b18..1c5eb7b9a9af 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -140,7 +140,7 @@ public void setTemplateSize(long templateSize) { } public Long getTemplateSize() { - return templateSize; + return templateSize == 0 ? templatePhySicalSize : templateSize; } public void setTemplatePhySicalSize(long templatePhySicalSize) { diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index 128652fc64fa..7d544c2e49c0 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -635,6 +635,19 @@ public String[] generateConfiguration(final LoadBalancerConfigCommand lbCmd) { if (lbCmd.keepAliveEnabled) { dSection.set(7, "\tno option httpclose"); } + if (lbCmd.idleTimeout > 0) { + dSection.set(9, "\ttimeout client " + Long.toString(lbCmd.idleTimeout)); + dSection.set(10, "\ttimeout server " + Long.toString(lbCmd.idleTimeout)); + } else if (lbCmd.idleTimeout == 0) { + // .remove() is not allowed, only .set() operations are allowed as the list + // is a fixed size. So lets just mark the entry as blank. + dSection.set(9, ""); + dSection.set(10, ""); + } else { + // Negative idleTimeout values are considered invalid; retain the + // default HAProxy timeout values from defaultsSection for predictability. + logger.warn("Negative idleTimeout ({}) configured; retaining default HAProxy timeouts.", lbCmd.idleTimeout); + } if (logger.isDebugEnabled()) { for (final String s : dSection) { diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java new file mode 100644 index 000000000000..34cf6d4ca34c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class CreateImageTransferAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public CreateImageTransferAnswer() { + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public CreateImageTransferAnswer(CreateImageTransferCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java new file mode 100644 index 000000000000..95b56c9a9c38 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -0,0 +1,94 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class CreateImageTransferCommand extends Command { + private String transferId; + private String exportName; + private String socket; + private String direction; + private String checkpointId; + private String file; + private ImageTransfer.Backend backend; + private int idleTimeoutSeconds; + + public CreateImageTransferCommand() { + } + + private CreateImageTransferCommand(String transferId, String direction, String socket, int idleTimeoutSeconds) { + this.transferId = transferId; + this.direction = direction; + this.socket = socket; + this.idleTimeoutSeconds = idleTimeoutSeconds; + } + + public CreateImageTransferCommand(String transferId, String direction, String exportName, String socket, String checkpointId, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); + this.backend = ImageTransfer.Backend.nbd; + this.exportName = exportName; + this.checkpointId = checkpointId; + } + + public CreateImageTransferCommand(String transferId, String direction, String socket, String file, int idleTimeoutSeconds) { + this(transferId, direction, socket, idleTimeoutSeconds); + if (direction == ImageTransfer.Direction.download.toString()) { + throw new IllegalArgumentException("File backend is only supported for upload"); + } + this.backend = ImageTransfer.Backend.file; + this.file = file; + } + + public String getExportName() { + return exportName; + } + + public String getSocket() { + return socket; + } + + public String getFile() { + return file; + } + + public ImageTransfer.Backend getBackend() { + return backend; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getDirection() { + return direction; + } + + public String getCheckpointId() { + return checkpointId; + } + + public int getIdleTimeoutSeconds() { + return idleTimeoutSeconds; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java new file mode 100644 index 000000000000..81cf6c1abfcc --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java @@ -0,0 +1,60 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; + +public class DeleteVmCheckpointCommand extends Command { + private String vmName; + private String checkpointId; + private Map diskPathUuidMap; + private boolean stoppedVM; + + public DeleteVmCheckpointCommand() { + } + + public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map diskPathUuidMap, boolean stoppedVM) { + this.vmName = vmName; + this.checkpointId = checkpointId; + this.diskPathUuidMap = diskPathUuidMap; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getCheckpointId() { + return checkpointId; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java new file mode 100644 index 000000000000..84d9b1ff8186 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/FinalizeImageTransferCommand.java @@ -0,0 +1,40 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class FinalizeImageTransferCommand extends Command { + private String transferId; + + public FinalizeImageTransferCommand() { + } + + public FinalizeImageTransferCommand(String transferId) { + this.transferId = transferId; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java new file mode 100644 index 000000000000..d7cbf097df90 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupAnswer.java @@ -0,0 +1,44 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartBackupAnswer extends Answer { + private Long checkpointCreateTime; + + public StartBackupAnswer() { + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartBackupAnswer(StartBackupCommand cmd, boolean success, String details, Long checkpointCreateTime) { + super(cmd, success, details); + this.checkpointCreateTime = checkpointCreateTime; + } + + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java new file mode 100644 index 000000000000..6f1ed6834500 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -0,0 +1,91 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Map; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class StartBackupCommand extends Command { + private String vmName; + private String toCheckpointId; + private String fromCheckpointId; + private Long fromCheckpointCreateTime; + private String socket; + private Map diskPathUuidMap; + private boolean stoppedVM; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map diskPathPassphraseMap; + + public StartBackupCommand() { + } + + public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, Long fromCheckpointCreateTime, + String socket, Map diskPathUuidMap, Map passphrases, boolean stoppedVM) { + this.vmName = vmName; + this.toCheckpointId = toCheckpointId; + this.fromCheckpointId = fromCheckpointId; + this.fromCheckpointCreateTime = fromCheckpointCreateTime; + this.socket = socket; + this.diskPathUuidMap = diskPathUuidMap; + this.diskPathPassphraseMap = passphrases; + this.stoppedVM = stoppedVM; + } + + public String getVmName() { + return vmName; + } + + public String getToCheckpointId() { + return toCheckpointId; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public Long getFromCheckpointCreateTime() { + return fromCheckpointCreateTime; + } + + public String getSocket() { + return socket; + } + + public Map getDiskPathUuidMap() { + return diskPathUuidMap; + } + + public boolean isIncremental() { + return fromCheckpointId != null && !fromCheckpointId.isEmpty(); + } + + public boolean isStoppedVM() { + return stoppedVM; + } + + public Map getDiskPathPassphraseMap() { + return diskPathPassphraseMap; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java new file mode 100644 index 000000000000..d8c78d3c8807 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerAnswer.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StartNBDServerAnswer extends Answer { + private String imageTransferId; + private String transferUrl; + + public StartNBDServerAnswer() { + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + public StartNBDServerAnswer(StartNBDServerCommand cmd, boolean success, String details, + String imageTransferId, String transferUrl) { + super(cmd, success, details); + this.imageTransferId = imageTransferId; + this.transferUrl = transferUrl; + } + + public String getImageTransferId() { + return imageTransferId; + } + + public void setImageTransferId(String imageTransferId) { + this.imageTransferId = imageTransferId; + } + + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java new file mode 100644 index 000000000000..07e50f11fd37 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -0,0 +1,78 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class StartNBDServerCommand extends Command { + private String transferId; + private String exportName; + private String volumePath; + private String socket; + private String direction; + private String fromCheckpointId; + @LogLevel(LogLevel.Log4jLevel.Off) + private byte[] passphrase; + + public StartNBDServerCommand() { + } + + protected StartNBDServerCommand(String transferId, String exportName, String volumePath, String socket, String direction, String fromCheckpointId, byte[] passphrase) { + this.transferId = transferId; + this.socket = socket; + this.exportName = exportName; + this.volumePath = volumePath; + this.direction = direction; + this.fromCheckpointId = fromCheckpointId; + this.passphrase = passphrase; + } + + public String getExportName() { + return exportName; + } + + public String getSocket() { + return socket; + } + + public String getTransferId() { + return transferId; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public String getVolumePath() { + return volumePath; + } + + public String getDirection() { + return direction; + } + + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public byte[] getPassphrase() { + return passphrase; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java new file mode 100644 index 000000000000..ce977f31e005 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupAnswer.java @@ -0,0 +1,30 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class StopBackupAnswer extends Answer { + + public StopBackupAnswer() { + } + + public StopBackupAnswer(StopBackupCommand cmd, boolean success, String details) { + super(cmd, success, details); + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java new file mode 100644 index 000000000000..d3055021e9de --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopBackupCommand.java @@ -0,0 +1,52 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopBackupCommand extends Command { + private String vmName; + private Long vmId; + private Long backupId; + + public StopBackupCommand() { + } + + public StopBackupCommand(String vmName, Long vmId, Long backupId) { + this.vmName = vmName; + this.vmId = vmId; + this.backupId = backupId; + } + + public String getVmName() { + return vmName; + } + + public Long getVmId() { + return vmId; + } + + public Long getBackupId() { + return backupId; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java new file mode 100644 index 000000000000..d75168a22eb2 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/StopNBDServerCommand.java @@ -0,0 +1,46 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; + +public class StopNBDServerCommand extends Command { + private String transferId; + private String direction; + + public StopNBDServerCommand() { + } + + public StopNBDServerCommand(String transferId, String direction) { + this.transferId = transferId; + this.direction = direction; + } + + public String getTransferId() { + return transferId; + } + + public String getDirection() { + return direction; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index a1485463eaa0..05619e5632b0 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -21,6 +21,7 @@ import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -33,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -128,15 +130,14 @@ public void setFollowRedirects(boolean followRedirects) { */ protected File createTemporaryDirectoryAndFile(String downloadDir) { createFolder(downloadDir); - return new File(downloadDir + File.separator + getFileNameFromUrl()); + return new File(downloadDir + File.separator + getTemporaryFileName()); } /** - * Return filename from url + * Return filename from the temporary download file */ - public String getFileNameFromUrl() { - String[] urlParts = url.split("/"); - return urlParts[urlParts.length - 1]; + public String getTemporaryFileName() { + return String.format("%s.%s", UUID.randomUUID(), FilenameUtils.getExtension(url)); } @Override diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java index 2050b9ef09f7..854c310cde9a 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -97,7 +97,7 @@ public Pair downloadTemplate() { DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(), getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath); try { - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getTemporaryFileName()); File f = new File(getDownloadedFilePath()); if (f.exists()) { f.delete(); diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 21184ef07fe9..6b0959b78ffb 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -69,7 +69,7 @@ public Pair downloadTemplate() { String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid); Script.runSimpleBashScript(mount); String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId()); - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getTemporaryFileName()); Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath()); Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return new Pair<>(true, getDownloadedFilePath()); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index 253a2607a72c..98db75f39025 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -19,6 +19,9 @@ package org.apache.cloudstack.storage.command; +import com.cloud.configuration.Resource; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; + public class TemplateOrVolumePostUploadCommand { long entityId; @@ -185,6 +188,11 @@ public void setDescription(String description) { this.description = description; } + public void setDefaultMaxSecondaryStorageInBytes(long defaultMaxSecondaryStorageInBytes) { + this.defaultMaxSecondaryStorageInGB = defaultMaxSecondaryStorageInBytes != Resource.RESOURCE_UNLIMITED ? + ByteScaleUtils.bytesToGibibytes(defaultMaxSecondaryStorageInBytes) : Resource.RESOURCE_UNLIMITED; + } + public void setDefaultMaxSecondaryStorageInGB(long defaultMaxSecondaryStorageInGB) { this.defaultMaxSecondaryStorageInGB = defaultMaxSecondaryStorageInGB; } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java index 9e6b76e467ff..f78744046f73 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java @@ -28,6 +28,7 @@ public enum EntityType { } private String entityUuid; private EntityType entityType; + private Boolean abort; protected UploadStatusCommand() { } @@ -37,6 +38,11 @@ public UploadStatusCommand(String entityUuid, EntityType entityType) { this.entityType = entityType; } + public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) { + this(entityUuid, entityType); + this.abort = abort; + } + public String getEntityUuid() { return entityUuid; } @@ -45,6 +51,10 @@ public EntityType getEntityType() { return entityType; } + public Boolean getAbort() { + return abort; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java index 6d4c6234c424..4ddadab999a4 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java @@ -235,7 +235,7 @@ protected LoadBalancerConfigCommand generateLoadBalancerConfigCommand() { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 0L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java index 4196587cc3f2..ed819bb7c68f 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java @@ -779,7 +779,7 @@ protected LoadBalancerConfigCommand generateLoadBalancerConfigCommand1() { final LoadBalancerTO[] arrayLbs = new LoadBalancerTO[lbs.size()]; lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; @@ -795,7 +795,7 @@ protected LoadBalancerConfigCommand generateLoadBalancerConfigCommand2() { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); nic.setIp("10.1.10.2"); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; diff --git a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java index 72361c2880ed..073f976719b5 100644 --- a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java +++ b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java @@ -79,13 +79,14 @@ public void testGenerateConfigurationLoadBalancerConfigCommand() { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("keepalive disabled should result in 'option httpclose' in the resulting haproxy config", result.contains("\toption httpclose")); - cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true); + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); result = genConfig(hpg, cmd); assertTrue("keepalive enabled should result in 'no option httpclose' in the resulting haproxy config", result.contains("\tno option httpclose")); + // TODO // create lb command // setup tests for @@ -93,6 +94,27 @@ public void testGenerateConfigurationLoadBalancerConfigCommand() { // httpmode } + /** + * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. + */ + @Test + public void testGenerateConfigurationLoadBalancerIdleTimeoutConfigCommand() { + LoadBalancerTO lb = new LoadBalancerTO("1", "10.2.0.1", 80, "http", "bla", false, false, false, null); + LoadBalancerTO[] lba = new LoadBalancerTO[1]; + lba[0] = lb; + HAProxyConfigurator hpg = new HAProxyConfigurator(); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); + String result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 0 should not generate 'timeout server' in the resulting haproxy config", !result.contains("\ttimeout server")); + assertTrue("idleTimeout of 0 should not generate 'timeout client' in the resulting haproxy config", !result.contains("\ttimeout client")); + + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 1234L); + result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 1234 should result in 'timeout server 1234' in the resulting haproxy config", result.contains("\ttimeout server 1234")); + assertTrue("idleTimeout of 1234 should result in 'timeout client 1234' in the resulting haproxy config", result.contains("\ttimeout client 1234")); + } + /** * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. */ @@ -106,7 +128,7 @@ public void testGenerateConfigurationLoadBalancerProxyProtocolConfigCommand() { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("'send-proxy' should result if protocol is 'tcp-proxy'", result.contains("send-proxy")); } @@ -118,7 +140,7 @@ public void generateConfigurationTestWithCidrList() { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("acl network_allowed src 1.1.1.1 2.2.2.2/24 \n\ttcp-request connection reject if !network_allowed")); } @@ -131,7 +153,7 @@ public void generateConfigurationTestWithSslCert() { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("bind 10.2.0.1:443 ssl crt /etc/cloudstack/ssl/10_2_0_1-443.pem")); } diff --git a/debian/changelog b/debian/changelog index 02251137e9d0..41f97748a0b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,13 +2,19 @@ cloudstack (4.23.0.0-SNAPSHOT) unstable; urgency=low * Update the version to 4.23.0.0-SNAPSHOT - -- the Apache CloudStack project Thu, 30 Oct 2025 19:23:55 +0530 + -- the Apache CloudStack project Fri, 22 May 2026 10:20:00 -0300 + +cloudstack (4.22.1.0) unstable; urgency=low + + * Update the version to 4.22.1.0 -cloudstack (4.23.0.0-SNAPSHOT-SNAPSHOT) unstable; urgency=low + -- the Apache CloudStack project Mon, 11 May 2026 20:26:07 +0530 - * Update the version to 4.23.0.0-SNAPSHOT-SNAPSHOT +cloudstack (4.22.0.0) unstable; urgency=low - -- the Apache CloudStack project Thu, Aug 28 11:58:36 2025 +0530 + * Update the version to 4.22.0.0 + + -- the Apache CloudStack project Thu, 30 Oct 2025 19:23:55 +0530 cloudstack (4.21.0.0) unstable; urgency=low diff --git a/debian/control b/debian/control index 2b8ce929c639..cdf663ef8906 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 57dc1b7bf728..5eca02f33d51 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -59,6 +59,8 @@ */ public interface VirtualMachineManager extends Manager { + String KVM_BLANK_VM_TEMPLATE_NAME = "kvm-blank-vm-template"; + ConfigKey ExecuteInSequence = new ConfigKey<>("Advanced", Boolean.class, "execute.in.sequence.hypervisor.commands", "false", "If set to true, start, stop, reboot, copy and migrate commands will be serialized on the agent side. If set to false the commands are executed in parallel. Default value is false.", false); @@ -181,15 +183,6 @@ void orchestrateStart(String vmUuid, Map pa void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); @@ -321,4 +314,8 @@ void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTemplate t ServiceOffering serviceOffering, Account systemAccount, DeploymentPlan plan) throws InsufficientServerCapacityException; + boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template); + + boolean isBlankInstance(VirtualMachineTemplate template); + } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 947cbd8e6182..030c1277efe2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -122,6 +122,14 @@ public interface NetworkOrchestrationService { "Load Balancer(haproxy) maximum number of concurrent connections(global max)", true, Scope.Global); + ConfigKey NETWORK_LB_HAPROXY_IDLE_TIMEOUT = new ConfigKey<>( + "Network", + Long.class, + "network.loadbalancer.haproxy.idle.timeout", + "50000", + "Load Balancer(haproxy) idle timeout in milliseconds. Use 0 for infinite.", + true, + Scope.Global); List setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault) throws ConcurrentOperationException; @@ -212,6 +220,11 @@ Network createGuestNetwork(long networkOfferingId, String name, String displayTe Boolean displayNetworkEnabled, String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + Network createGuestNetwork(long networkOfferingId, String name, String displayText, String gateway, String cidr, String vlanId, boolean bypassVlanOverlapCheck, String networkDomain, Account owner, + Long domainId, PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Boolean subdomainAccess, Long vpcId, String ip6Gateway, String ip6Cidr, + Boolean displayNetworkEnabled, String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, + String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize, boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + UserDataServiceProvider getPasswordResetProvider(Network network); UserDataServiceProvider getSSHKeyResetProvider(Network network); @@ -310,7 +323,7 @@ void implementNetworkElementsAndResources(DeployDestination dest, ReservationCon void removeDhcpServiceInSubnet(Nic nic); - boolean resourceCountNeedsUpdate(NetworkOffering ntwkOff, ACLType aclType); + boolean isResourceCountUpdateNeeded(NetworkOffering networkOffering); void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 6f8c46304567..a55219511cb3 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -120,7 +120,7 @@ VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPoolPodId, void destroyVolume(Volume volume); DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId); + Account owner, Long deviceId, boolean incrementResourceCount); VolumeInfo createVolumeOnPrimaryStorage(VirtualMachine vm, VolumeInfo volume, HypervisorType rootDiskHyperType, StoragePool storagePool) throws NoTransitionException; diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 4c81c7359f25..abaf6ea967d0 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -133,6 +133,20 @@ public interface CapacityManager { "capacity.calculate.workers", "1", "Number of worker threads to be used for capacities calculation", true); + ConfigKey KvmMemoryDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "kvm.memory.dynamic.scaling.capacity", "0", + "Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. " + + "The 'kvm.memory.dynamic.scaling.capacity' setting's value will be used to define the value of the " + + "'' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's memory capacity will be considered.", + true, ConfigKey.Scope.Cluster); + + ConfigKey KvmCpuDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "kvm.cpu.dynamic.scaling.capacity", "0", + "Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. " + + "The 'kvm.cpu.dynamic.scaling.capacity' setting's value will be used to define the value of the " + + "'' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's CPU cores capacity will be considered.", + true, ConfigKey.Scope.Cluster); + public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost); diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index ddc8153d7398..53bfcce27038 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -21,6 +21,7 @@ import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.component.Manager; import com.cloud.vm.VMInstanceVO; import org.apache.cloudstack.framework.config.ConfigKey; @@ -32,6 +33,8 @@ */ public interface HighAvailabilityManager extends Manager { + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); @@ -72,10 +75,10 @@ public interface HighAvailabilityManager extends Manager { + " which are registered for the HA event that were successful and are now ready to be purged.", true, Cluster); - public static final ConfigKey KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false", + ConfigKey KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false", "Proceed fencing the host even the heartbeat failed for only one storage pool", false, ConfigKey.Scope.Zone); - public enum WorkType { + enum WorkType { Migration, // Migrating VMs off of a host. Stop, // Stops a VM for storage pool migration purposes. This should be obsolete now. CheckStop, // Checks if a VM has been stopped. diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index e724f5d081bd..4767e86e8ab8 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -122,6 +122,8 @@ public interface ResourceManager extends ResourceService, Configurable { public boolean executeUserRequest(long hostId, ResourceState.Event event) throws AgentUnavailableException; + boolean executeUserRequest(long hostId, ResourceState.Event event, boolean isForced, boolean isForceDeleteStorage) throws AgentUnavailableException; + boolean resourceStateTransitTo(Host host, Event event, long msId) throws NoTransitionException; boolean umanageHost(long hostId); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 527c5259376a..1215829d92f8 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -805,8 +805,11 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); + String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT); + String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR); + String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { _hostDao.loadDetails(host); boolean updateNeeded = false; if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { @@ -821,6 +824,26 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); updateNeeded = true; } + if (StringUtils.isNotBlank(vddkSupport) && !vddkSupport.equals(host.getDetails().get(Host.HOST_VDDK_SUPPORT))) { + host.getDetails().put(Host.HOST_VDDK_SUPPORT, vddkSupport); + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkLibDir).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_LIB_DIR)))) { + if (StringUtils.isBlank(vddkLibDir)) { + host.getDetails().remove(Host.HOST_VDDK_LIB_DIR); + } else { + host.getDetails().put(Host.HOST_VDDK_LIB_DIR, vddkLibDir); + } + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkVersion).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_VERSION)))) { + if (StringUtils.isBlank(vddkVersion)) { + host.getDetails().remove(Host.HOST_VDDK_VERSION); + } else { + host.getDetails().put(Host.HOST_VDDK_VERSION, vddkVersion); + } + updateNeeded = true; + } if (updateNeeded) { _hostDao.saveDetails(host); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index cfa0949883fd..38a198b73040 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -1306,11 +1306,20 @@ public String dispatch(final ClusterServicePdu pdu) { boolean result; try { - result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent()); + result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent(), cmd.isForced(), cmd.isForceDeleteStorage()); logger.debug("Result is {}", result); } catch (final AgentUnavailableException ex) { logger.warn("Agent is unavailable", ex); return null; + } catch (final RuntimeException ex) { + logger.error(String.format("Failed to execute propagated event %s for host %d", cmd.getEvent().name(), cmd.getHostId()), ex); + final Answer[] answers = new Answer[1]; + String details = ex.getMessage(); + if (details == null || details.isEmpty()) { + details = ex.toString(); + } + answers[0] = new Answer(cmd, false, details); + return _gson.toJson(answers); } final Answer[] answers = new Answer[1]; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index b20c06fc2c31..db50a5134d8d 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -50,7 +50,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; - +import com.cloud.hypervisor.KVMGuru; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -303,14 +303,13 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.gson.Gson; - public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { public static final String VM_WORK_JOB_HANDLER = VirtualMachineManagerImpl.class.getSimpleName(); @@ -577,7 +576,13 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t logger.debug("Allocating disks for {}", persistedVm); - allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + if (isBlankInstance(template)) { + logger.debug("Template is a dummy template for hypervisor {}, skipping volume allocation", hyperType); + return; + } else { + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot); + } + // Create new Volume context and inject event resource type, id and details to generate VOLUME.CREATE event for the ROOT disk. CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume); @@ -588,7 +593,7 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t Long deviceId = dataDiskDeviceIds.get(index++); String volumeName = deviceId == null ? "DATA-" + persistedVm.getId() : "DATA-" + persistedVm.getId() + "-" + String.valueOf(deviceId); volumeMgr.allocateRawVolume(Type.DATADISK, volumeName, dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId); + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId, true); } } if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { @@ -598,7 +603,7 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf( diskNumber), diskOffering, diskOfferingSize, null, null, - persistedVm, dataDiskTemplate, owner, diskNumber); + persistedVm, dataDiskTemplate, owner, diskNumber, true); diskNumber++; } } @@ -628,7 +633,7 @@ private void allocateRootVolume(VMInstanceVO vm, VirtualMachineTemplate template String rootVolumeName = String.format("ROOT-%s", vm.getId()); if (template.getFormat() == ImageFormat.ISO) { volumeMgr.allocateRawVolume(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), - rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null); + rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null, true); } else if (Arrays.asList(ImageFormat.BAREMETAL, ImageFormat.EXTERNAL).contains(template.getFormat())) { logger.debug("{} has format [{}]. Skipping ROOT volume [{}] allocation.", template, template.getFormat(), rootVolumeName); } else { @@ -2219,7 +2224,6 @@ private List> getVolumesToDisconnect(VirtualMachine vm) { protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) { final VirtualMachine vm = profile.getVirtualMachine(); Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup); updateStopCommandForExternalHypervisorType(vm.getHypervisorType(), profile, stpCmd); if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { @@ -4020,19 +4024,6 @@ protected void runInContext() { } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -4472,11 +4463,6 @@ public void checkIfCanUpgrade(final VirtualMachine vmInstance, final ServiceOffe throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { @@ -5183,7 +5169,7 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old try { result = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); } catch (Exception ex) { - throw new RuntimeException("Unhandled exception", ex); + throw new RuntimeException("Unable to reconfigure VM.", ex); } if (result != null) { @@ -5196,22 +5182,29 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering oldServiceOffering, ServiceOffering newServiceOffering, boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException { - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + VMInstanceVO vm = _vmDao.findByUuid(vmUuid); HostVO hostVo = _hostDao.findById(vm.getHostId()); - Long clustedId = hostVo.getClusterId(); - Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clustedId); - Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clustedId); - boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clustedId); - boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clustedId); + Long clusterId = hostVo.getClusterId(); + Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clusterId); + Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clusterId); + boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clusterId); + boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clusterId); int minMemory = (int)(newServiceOffering.getRamSize() / (divideMemoryByOverprovisioning ? memoryOvercommitRatio : 1)); int minSpeed = (int)(newServiceOffering.getSpeed() / (divideCpuByOverprovisioning ? cpuOvercommitRatio : 1)); - ScaleVmCommand scaleVmCommand = - new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, - newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse()); + Double cpuQuotaPercentage = null; + if (newServiceOffering.getLimitCpuUse() && vm.getHypervisorType().equals(HypervisorType.KVM)) { + KVMGuru kvmGuru = (KVMGuru) _hvGuruMgr.getGuru(vm.getHypervisorType()); + cpuQuotaPercentage = kvmGuru.getCpuQuotaPercentage(minSpeed, hostVo.getSpeed()); + } + + boolean limitCpuUseChange = oldServiceOffering.getLimitCpuUse() != newServiceOffering.getLimitCpuUse(); + ScaleVmCommand scaleVmCommand = new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, newServiceOffering.getSpeed(), + minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, + newServiceOffering.getLimitCpuUse(), cpuQuotaPercentage, limitCpuUseChange); scaleVmCommand.getVirtualMachine().setId(vm.getId()); scaleVmCommand.getVirtualMachine().setUuid(vm.getUuid()); @@ -5240,16 +5233,20 @@ private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering old throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails())); } - upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering); + if (reconfiguringOnExistingHost) { + _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); + } + + boolean vmUpgraded = upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering); + if (vmUpgraded) { + vm = _vmDao.findById(vm.getId()); + } if (vm.getType().equals(VirtualMachine.Type.User)) { _userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE); } if (reconfiguringOnExistingHost) { - vm.setServiceOfferingId(oldServiceOffering.getId()); - _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); - vm.setServiceOfferingId(newServiceOffering.getId()); _capacityMgr.allocateVmCapacity(vm, false); } @@ -5278,9 +5275,20 @@ private void removeCustomOfferingDetails(long vmId) { private void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering) { Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); - details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); - details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); - details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); + + // We need to restore only the customizable parameters. If we save a parameter that is not customizable and attempt + // to restore a VM snapshot, com.cloud.vm.UserVmManagerImpl.validateCustomParameters will fail. + ServiceOffering unfilledOffering = _serviceOfferingDao.findByIdIncludingRemoved(serviceOffering.getId()); + if (unfilledOffering.getCpu() == null) { + details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); + } + if (unfilledOffering.getSpeed() == null) { + details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); + } + if (unfilledOffering.getRamSize() == null) { + details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); + } + List detailList = new ArrayList<>(); for (Map.Entry entry: details.entrySet()) { VMInstanceDetailVO detailVO = new VMInstanceDetailVO(vmId, entry.getKey(), entry.getValue(), true); @@ -6706,4 +6714,18 @@ public void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTem vmProfile), DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); } } + + @Override + public boolean isBlankInstanceDefaultTemplate(VirtualMachineTemplate template) { + return KVM_BLANK_VM_TEMPLATE_NAME.equals(template.getUniqueName()); + } + + @Override + public boolean isBlankInstance(VirtualMachineTemplate template) { + if (isBlankInstanceDefaultTemplate(template)) { + return true; + } + return Boolean.TRUE.equals( + MapUtils.getBoolean(CallContext.current().getContextParameters(), ApiConstants.BLANK_INSTANCE)); + } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 8639f006383f..964265cb873e 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.orchestration; import com.cloud.storage.Snapshot; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.template.VirtualMachineTemplate; import java.net.URL; @@ -292,9 +293,11 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow ServiceOfferingVO computeOffering = _serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + VMTemplateVO iso = _templateDao.findByIdIncludingRemoved(Long.valueOf(isoId)); + DiskOfferingInfo rootDiskOfferingInfo = new DiskOfferingInfo(); - if (diskOfferingId == null) { + if (diskOfferingId == null && !_itMgr.isBlankInstance(iso)) { throw new InvalidParameterValueException("Installing from ISO requires a disk offering to be specified for the root disk."); } DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); @@ -345,7 +348,7 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow HypervisorType hypervisorType = HypervisorType.valueOf(hypervisor); - _itMgr.allocate(vm.getInstanceName(), _templateDao.findByIdIncludingRemoved(new Long(isoId)), computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, + _itMgr.allocate(vm.getInstanceName(), iso, computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, networkIpMap, plan, hypervisorType, extraDhcpOptionMap, null, volume, snapshot); return vmEntity; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 77ea965e50aa..4262ee701aab 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -58,6 +58,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.cloudstack.network.dao.NetworkPermissionDao; +import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; @@ -86,6 +87,7 @@ import com.cloud.api.query.vo.DomainRouterJoinVO; import com.cloud.bgp.BGPService; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.ASNumberVO; import com.cloud.dc.ClusterVO; @@ -214,6 +216,7 @@ import com.cloud.offerings.dao.NetworkOfferingDetailsDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; import com.cloud.resource.ResourceManager; +import com.cloud.resourcelimit.CheckedReservation; import com.cloud.server.ManagementServer; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; @@ -447,6 +450,8 @@ public void setDhcpProviders(final List dhcpProviders) { ClusterDao clusterDao; @Inject RoutedIpv4Manager routedIpv4Manager; + @Inject + private ReservationDao reservationDao; protected StateMachine2 _stateMachine; ScheduledExecutorService _executor; @@ -858,6 +863,7 @@ private static NetworkVO getNetworkVO(long id, final NetworkOffering offering, f vpcId, offering.isRedundantRouter(), predefined.getExternalId()); vo.setDisplayNetwork(isDisplayNetworkEnabled == null || isDisplayNetworkEnabled); vo.setStrechedL2Network(offering.isSupportingStrechedL2()); + vo.setKeepMacAddressOnPublicNic(predefined.getKeepMacAddressOnPublicNic()); return vo; } @@ -2719,7 +2725,7 @@ public Network createPrivateNetwork(final long networkOfferingId, final String n return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, null, owner, null, pNtwk, pNtwk.getDataCenterId(), ACLType.Account, null, vpcId, null, null, true, null, null, null, true, null, null, - null, null, null, null, null, null); + null, null, null, null, null, null, true); } @Override @@ -2730,10 +2736,25 @@ public Network createGuestNetwork(final long networkOfferingId, final String nam final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, + networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, + isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, + ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize, true); + } + + @Override + @DB + public Network createGuestNetwork(final long networkOfferingId, final String name, final String displayText, final String gateway, final String cidr, String vlanId, + boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, + final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, + final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, + String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, + Pair vrIfaceMTUs, Integer networkCidrSize, boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { // create Isolated/Shared/L2 network return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, - isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize); + isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, + ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize, keepMacAddressOnPublicNic); } @DB @@ -2742,7 +2763,8 @@ private Network createGuestNetwork(final long networkOfferingId, final String na final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork, String routerIp, String routerIpv6, final String ip4Dns1, final String ip4Dns2, - final String ip6Dns1, final String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + final String ip6Dns1, final String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize, + boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { final NetworkOfferingVO ntwkOff = _networkOfferingDao.findById(networkOfferingId); final DataCenterVO zone = _dcDao.findById(zoneId); @@ -2752,12 +2774,6 @@ private Network createGuestNetwork(final long networkOfferingId, final String na return null; } - final boolean updateResourceCount = resourceCountNeedsUpdate(ntwkOff, aclType); - //check resource limits - if (updateResourceCount) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.network, isDisplayNetworkEnabled); - } - // Validate network offering if (ntwkOff.getState() != NetworkOffering.State.Enabled) { // see NetworkOfferingVO @@ -2776,218 +2792,219 @@ private Network createGuestNetwork(final long networkOfferingId, final String na boolean ipv6 = false; - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - ipv6 = true; - } - // Validate zone - if (zone.getNetworkType() == NetworkType.Basic) { - // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true - if (aclType == null || aclType != ACLType.Domain) { - throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); - } - - // Only one guest network is supported in Basic zone - final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); - if (!guestNetworks.isEmpty()) { - throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); + try (CheckedReservation networkReservation = new CheckedReservation(owner, domainId, Resource.ResourceType.network, null, null, 1L, reservationDao, _resourceLimitMgr)) { + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + ipv6 = true; } + // Validate zone + if (zone.getNetworkType() == NetworkType.Basic) { + // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true + if (aclType == null || aclType != ACLType.Domain) { + throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + } - // if zone is basic, only Shared network offerings w/o source nat service are allowed - if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { - throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " - + Service.SourceNat.getName() + " service are allowed"); - } + // Only one guest network is supported in Basic zone + final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); + if (!guestNetworks.isEmpty()) { + throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); + } - if (domainId == null || domainId != Domain.ROOT_DOMAIN) { - throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); - } + // if zone is basic, only Shared network offerings w/o source nat service are allowed + if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { + throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " + + Service.SourceNat.getName() + " service are allowed"); + } - if (subdomainAccess == null) { - subdomainAccess = true; - } else if (!subdomainAccess) { - throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); - } + if (domainId == null || domainId != Domain.ROOT_DOMAIN) { + throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); + } - if (vlanId == null) { - vlanId = Vlan.UNTAGGED; - } else { - if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { - throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + if (subdomainAccess == null) { + subdomainAccess = true; + } else if (!subdomainAccess) { + throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); } - } - } else if (zone.getNetworkType() == NetworkType.Advanced) { - if (zone.isSecurityGroupEnabled()) { - if (isolatedPvlan != null) { - throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); + if (vlanId == null) { + vlanId = Vlan.UNTAGGED; + } else { + if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { + throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + } } - // Only Account specific Isolated network with sourceNat service disabled are allowed in security group - // enabled zone - if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { - throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); + + } else if (zone.getNetworkType() == NetworkType.Advanced) { + if (zone.isSecurityGroupEnabled()) { + if (isolatedPvlan != null) { + throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); + } + // Only Account specific Isolated network with sourceNat service disabled are allowed in security group + // enabled zone + if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { + throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); + } + if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { + throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + } } - if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { - throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + + //don't allow eip/elb networks in Advance zone + if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { + throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); } } - //don't allow eip/elb networks in Advance zone - if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { - throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); + if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { + _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); } - } - - if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { - _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); - } - //TODO(VXLAN): Support VNI specified - // VlanId can be specified only when network offering supports it - final boolean vlanSpecified = vlanId != null; - if (vlanSpecified != ntwkOff.isSpecifyVlan()) { - if (vlanSpecified) { - if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + //TODO(VXLAN): Support VNI specified + // VlanId can be specified only when network offering supports it + final boolean vlanSpecified = vlanId != null; + if (vlanSpecified != ntwkOff.isSpecifyVlan()) { + if (vlanSpecified) { + if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + } + } else { + throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); } - } else { - throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); } - } - if (vlanSpecified) { - URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); - // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks - URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; - if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - bypassVlanOverlapCheck = true; - } - //don't allow to specify vlan tag used by physical network for dynamic vlan allocation - if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) - && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " - + zone.getName()); - } - if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && - _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { - throw new InvalidParameterValueException(String.format( - "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", - isolatedPvlan, zone)); - } - if (!UuidUtils.isUuid(vlanId)) { - // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone - if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { - if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - vlanId, zone)); - } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - isolatedPvlan, zone)); - } else { - final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); - //for the network that is created as part of private gateway, - //the vnet is not coming from the data center vnet table, so the list can be empty - if (!dcVnets.isEmpty()) { - final DataCenterVnetVO dcVnet = dcVnets.get(0); - // Fail network creation if specified vlan is dedicated to a different account - if (dcVnet.getAccountGuestVlanMapId() != null) { - final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); - final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); - if (map.getAccountId() != owner.getAccountId()) { - throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); - } - // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool - } else { - final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); - if (maps != null && !maps.isEmpty()) { - final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); - final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); - if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { - throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " - + owner.getAccountName()); + if (vlanSpecified) { + URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); + // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks + URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; + if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + bypassVlanOverlapCheck = true; + } + //don't allow to specify vlan tag used by physical network for dynamic vlan allocation + if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) + && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { + throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + + zone.getName()); + } + if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && + _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { + throw new InvalidParameterValueException(String.format( + "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", + isolatedPvlan, zone)); + } + if (!UuidUtils.isUuid(vlanId)) { + // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone + if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { + if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + vlanId, zone)); + } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + isolatedPvlan, zone)); + } else { + final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); + //for the network that is created as part of private gateway, + //the vnet is not coming from the data center vnet table, so the list can be empty + if (!dcVnets.isEmpty()) { + final DataCenterVnetVO dcVnet = dcVnets.get(0); + // Fail network creation if specified vlan is dedicated to a different account + if (dcVnet.getAccountGuestVlanMapId() != null) { + final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); + final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); + if (map.getAccountId() != owner.getAccountId()) { + throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); + } + // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool + } else { + final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); + if (maps != null && !maps.isEmpty()) { + final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); + final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); + if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { + throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " + + owner.getAccountName()); + } } } } } - } - } else { - // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or - // shared network with same Vlan ID in the zone - if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { - throw new InvalidParameterValueException(String.format( - "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + } else { + // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or + // shared network with same Vlan ID in the zone + if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + throw new InvalidParameterValueException(String.format( + "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + } } } - } - } + } - // If networkDomain is not specified, take it from the global configuration - if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { - final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), - Service.Dns); - final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); - if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { - if (networkDomain != null) { - // TBD: NetworkOfferingId and zoneId. Send uuids instead. - throw new InvalidParameterValueException(String.format( - "Domain name change is not supported by network offering id=%d in zone %s", - networkOfferingId, zone)); - } - } else { - if (networkDomain == null) { - // 1) Get networkDomain from the corresponding account/domain/zone - if (aclType == ACLType.Domain) { - networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); - } else if (aclType == ACLType.Account) { - networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); + // If networkDomain is not specified, take it from the global configuration + if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { + final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), + Service.Dns); + final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); + if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { + if (networkDomain != null) { + // TBD: NetworkOfferingId and zoneId. Send uuids instead. + throw new InvalidParameterValueException(String.format( + "Domain name change is not supported by network offering id=%d in zone %s", + networkOfferingId, zone)); } - - // 2) If null, generate networkDomain using domain suffix from the global config variables + } else { if (networkDomain == null) { - networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); - } + // 1) Get networkDomain from the corresponding account/domain/zone + if (aclType == ACLType.Domain) { + networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); + } else if (aclType == ACLType.Account) { + networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); + } - } else { - // validate network domain - if (!NetUtils.verifyDomainName(networkDomain)) { - throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " - + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " - + "and the hyphen ('-'); can't start or end with \"-\""); + // 2) If null, generate networkDomain using domain suffix from the global config variables + if (networkDomain == null) { + networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); + } + + } else { + // validate network domain + if (!NetUtils.verifyDomainName(networkDomain)) { + throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " + + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " + + "and the hyphen ('-'); can't start or end with \"-\""); + } } } } - } - // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x - // limitation, remove after we introduce support for multiple ip ranges - // with different Cidrs for the same Shared network - final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced - && ntwkOff.getTrafficType() == TrafficType.Guest - && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); - if (cidr == null && ip6Cidr == null && cidrRequired) { - if (ntwkOff.getGuestType() == GuestType.Shared) { - throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); - } else { - throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x + // limitation, remove after we introduce support for multiple ip ranges + // with different Cidrs for the same Shared network + final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced + && ntwkOff.getTrafficType() == TrafficType.Guest + && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); + if (cidr == null && ip6Cidr == null && cidrRequired) { + if (ntwkOff.getGuestType() == GuestType.Shared) { + throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); + } else { + throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + } } - } - checkL2OfferingServices(ntwkOff); + checkL2OfferingServices(ntwkOff); - // No cidr can be specified in Basic zone - if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { - throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); - } + // No cidr can be specified in Basic zone + if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { + throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); + } - // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 - if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && - !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { + // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 + if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && + !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { throw new InvalidParameterValueException("Virtual Guest Cidr " + cidr + " is not RFC 1918 or 6598 compliant"); - } + } final String networkDomainFinal = networkDomain; final String vlanIdFinal = vlanId; @@ -3003,75 +3020,75 @@ public Network doInTransaction(final TransactionStatus status) { final NetworkVO userNetwork = new NetworkVO(); userNetwork.setNetworkDomain(networkDomainFinal); - if (cidr != null && gateway != null) { - userNetwork.setCidr(cidr); - userNetwork.setGateway(gateway); - } - - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - userNetwork.setIp6Cidr(ip6Cidr); - userNetwork.setIp6Gateway(ip6Gateway); - } + if (cidr != null && gateway != null) { + userNetwork.setCidr(cidr); + userNetwork.setGateway(gateway); + } - if (externalId != null) { - userNetwork.setExternalId(externalId); - } + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + userNetwork.setIp6Cidr(ip6Cidr); + userNetwork.setIp6Gateway(ip6Gateway); + } - if (StringUtils.isNotBlank(routerIp)) { - userNetwork.setRouterIp(routerIp); - } + if (externalId != null) { + userNetwork.setExternalId(externalId); + } - if (StringUtils.isNotBlank(routerIpv6)) { - userNetwork.setRouterIpv6(routerIpv6); - } + if (StringUtils.isNotBlank(routerIp)) { + userNetwork.setRouterIp(routerIp); + } - if (vrIfaceMTUs != null) { - if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { - userNetwork.setPublicMtu(vrIfaceMTUs.first()); - } else { - userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); + if (StringUtils.isNotBlank(routerIpv6)) { + userNetwork.setRouterIpv6(routerIpv6); } - if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { - userNetwork.setPrivateMtu(vrIfaceMTUs.second()); + if (vrIfaceMTUs != null) { + if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { + userNetwork.setPublicMtu(vrIfaceMTUs.first()); + } else { + userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); + } + + if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { + userNetwork.setPrivateMtu(vrIfaceMTUs.second()); + } else { + userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); + } } else { + userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); } - } else { - userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); - } - if (!GuestType.L2.equals(userNetwork.getGuestType())) { - if (StringUtils.isNotBlank(ip4Dns1)) { - userNetwork.setDns1(ip4Dns1); - } - if (StringUtils.isNotBlank(ip4Dns2)) { - userNetwork.setDns2(ip4Dns2); - } - if (StringUtils.isNotBlank(ip6Dns1)) { - userNetwork.setIp6Dns1(ip6Dns1); - } - if (StringUtils.isNotBlank(ip6Dns2)) { - userNetwork.setIp6Dns2(ip6Dns2); + if (!GuestType.L2.equals(userNetwork.getGuestType())) { + if (StringUtils.isNotBlank(ip4Dns1)) { + userNetwork.setDns1(ip4Dns1); + } + if (StringUtils.isNotBlank(ip4Dns2)) { + userNetwork.setDns2(ip4Dns2); + } + if (StringUtils.isNotBlank(ip6Dns1)) { + userNetwork.setIp6Dns1(ip6Dns1); + } + if (StringUtils.isNotBlank(ip6Dns2)) { + userNetwork.setIp6Dns2(ip6Dns2); + } } - } - if (vlanIdFinal != null) { - if (isolatedPvlan == null) { - URI uri = null; - if (UuidUtils.isUuid(vlanIdFinal)) { - //Logical router's UUID provided as VLAN_ID - userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field - } else { - uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); - } + if (vlanIdFinal != null) { + if (isolatedPvlan == null) { + URI uri = null; + if (UuidUtils.isUuid(vlanIdFinal)) { + //Logical router's UUID provided as VLAN_ID + userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field + } else { + uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + } - if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", - vlanIdFinal, zone)); - } + if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", + vlanIdFinal, zone)); + } userNetwork.setBroadcastUri(uri); if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { @@ -3095,6 +3112,7 @@ public Network doInTransaction(final TransactionStatus status) { } } userNetwork.setNetworkCidrSize(networkCidrSize); + userNetwork.setKeepMacAddressOnPublicNic(keepMacAddressOnPublicNic); final List networks = setupNetwork(owner, ntwkOff, userNetwork, plan, name, displayText, true, domainId, aclType, subdomainAccessFinal, vpcId, isDisplayNetworkEnabled); Network network; @@ -3115,8 +3133,8 @@ public Network doInTransaction(final TransactionStatus status) { } } - if (updateResourceCount) { - _resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.network, isDisplayNetworkEnabled); + if (isResourceCountUpdateNeeded(ntwkOff)) { + changeAccountResourceCountOrRecalculateDomainResourceCount(owner.getAccountId(), domainId, isDisplayNetworkEnabled, true); } UsageEventUtils.publishNetworkCreation(network); @@ -3127,6 +3145,7 @@ public Network doInTransaction(final TransactionStatus status) { CallContext.current().setEventDetails("Network ID: " + network.getUuid()); CallContext.current().putContextParameter(Network.class, network.getUuid()); return network; + } } @Override @@ -3492,9 +3511,8 @@ public List doInTransaction(TransactionStatus status) { } final NetworkOffering ntwkOff = _entityMgr.findById(NetworkOffering.class, networkFinal.getNetworkOfferingId()); - final boolean updateResourceCount = resourceCountNeedsUpdate(ntwkOff, networkFinal.getAclType()); - if (updateResourceCount) { - _resourceLimitMgr.decrementResourceCount(networkFinal.getAccountId(), ResourceType.network, networkFinal.getDisplayNetwork()); + if (isResourceCountUpdateNeeded(ntwkOff)) { + changeAccountResourceCountOrRecalculateDomainResourceCount(networkFinal.getAccountId(), networkFinal.getDomainId(), networkFinal.getDisplayNetwork(), false); } } return deletedVlans.second(); @@ -3517,6 +3535,23 @@ public List doInTransaction(TransactionStatus status) { return success; } + /** + * If it is a shared network with {@link ACLType#Domain}, it will belong to account {@link Account#ACCOUNT_ID_SYSTEM} and the resources will be not incremented for the + * domain. Therefore, we force the recalculation of the domain's resource count in this case. Otherwise, it will change the count for the account owner. + * @param incrementAccountResourceCount If true, the account resource count will be incremented by 1; otherwise, it will decremented by 1. + */ + private void changeAccountResourceCountOrRecalculateDomainResourceCount(Long accountId, Long domainId, boolean displayNetwork, boolean incrementAccountResourceCount) { + if (Account.ACCOUNT_ID_SYSTEM == accountId && ObjectUtils.isNotEmpty(domainId)) { + _resourceLimitMgr.recalculateDomainResourceCount(domainId, ResourceType.network, null); + } else { + if (incrementAccountResourceCount) { + _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.network, displayNetwork); + } else { + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.network, displayNetwork); + } + } + } + private void publishDeletedVlanRanges(List deletedVlanRangeToPublish) { if (CollectionUtils.isNotEmpty(deletedVlanRangeToPublish)) { for (VlanVO vlan : deletedVlanRangeToPublish) { @@ -3526,10 +3561,8 @@ private void publishDeletedVlanRanges(List deletedVlanRangeToPublish) { } @Override - public boolean resourceCountNeedsUpdate(final NetworkOffering ntwkOff, final ACLType aclType) { - //Update resource count only for Isolated account specific non-system networks - final boolean updateResourceCount = ntwkOff.getGuestType() == GuestType.Isolated && !ntwkOff.isSystemOnly() && aclType == ACLType.Account; - return updateResourceCount; + public boolean isResourceCountUpdateNeeded(NetworkOffering networkOffering) { + return !networkOffering.isSystemOnly(); } protected Pair> deleteVlansInNetwork(final NetworkVO network, final long userId, final Account callerAccount) { @@ -4924,6 +4957,7 @@ public ConfigKey[] getConfigKeys() { return new ConfigKey[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes, GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled, - TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN}; + TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN, + NETWORK_LB_HAPROXY_IDLE_TIMEOUT}; } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index e8c75afa81c5..bf3985d3ce77 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -40,6 +40,7 @@ import com.cloud.deploy.DeploymentClusterPlanner; import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ReservationHelper; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; @@ -82,6 +83,7 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.resourcelimit.Reserver; import org.apache.cloudstack.secret.PassphraseVO; import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.snapshot.SnapshotHelper; @@ -862,7 +864,7 @@ protected DiskProfile toDiskProfile(Volume vol, DiskOffering offering) { @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", create = true) @Override public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId) { + Long deviceId, boolean incrementResourceCount) { if (size == null) { size = offering.getDiskSize(); } else { @@ -901,7 +903,7 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri saveVolumeDetails(offering.getId(), vol.getId()); // Save usage event and update resource count for user vm volumes - if (vm.getType() == VirtualMachine.Type.User) { + if (vm.getType() == VirtualMachine.Type.User && incrementResourceCount) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size, Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); @@ -1938,14 +1940,20 @@ protected void updateVolumeSize(DataStore store, VolumeVO vol) throws ResourceAl template == null ? null : template.getSize(), vol.getPassphraseId() != null); - if (newSize != vol.getSize()) { - DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + if (newSize == vol.getSize()) { + return; + } + + DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + + List reservations = new ArrayList<>(); + try { VMInstanceVO vm = vol.getInstanceId() != null ? vmInstanceDao.findById(vol.getInstanceId()) : null; if (vm == null || vm.getType() == VirtualMachine.Type.User) { // Update resource count for user vm volumes when volume is attached if (newSize > vol.getSize()) { _resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()), - vol.isDisplay(), newSize - vol.getSize(), diskOffering); + vol.isDisplay(), newSize - vol.getSize(), diskOffering, reservations); _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), newSize - vol.getSize(), diskOffering); } else { @@ -1953,9 +1961,11 @@ protected void updateVolumeSize(DataStore store, VolumeVO vol) throws ResourceAl vol.getSize() - newSize, diskOffering); } } - vol.setSize(newSize); - _volsDao.persist(vol); + } finally { + ReservationHelper.closeAll(reservations); } + vol.setSize(newSize); + _volsDao.persist(vol); } @Override diff --git a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml index 17c5002c718b..49c668f50e8b 100644 --- a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml +++ b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml @@ -88,6 +88,7 @@ + diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java index 01afd0780f7e..e4047cf7973d 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java @@ -27,6 +27,6 @@ public interface AccountVlanMapDao extends GenericDao { public List listAccountVlanMapsByVlan(long vlanDbId); - public AccountVlanMapVO findAccountVlanMap(long accountId, long vlanDbId); + public AccountVlanMapVO findAccountVlanMap(Long accountId, long vlanDbId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java index 12114770f112..0844bb77caa2 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java @@ -48,9 +48,9 @@ public List listAccountVlanMapsByVlan(long vlanDbId) { } @Override - public AccountVlanMapVO findAccountVlanMap(long accountId, long vlanDbId) { + public AccountVlanMapVO findAccountVlanMap(Long accountId, long vlanDbId) { SearchCriteria sc = AccountVlanSearch.create(); - sc.setParameters("accountId", accountId); + sc.setParametersIfNotNull("accountId", accountId); sc.setParameters("vlanDbId", vlanDbId); return findOneIncludingRemovedBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index 6cfd2608f5de..3d04f6679def 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -23,6 +23,7 @@ import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; public interface ClusterDao extends GenericDao { @@ -61,4 +62,6 @@ public interface ClusterDao extends GenericDao { List listDistinctStorageAccessGroups(String name, String keyword); List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, HypervisorType hypervisorType, CPU.CPUArch arch); + + List listByZonesAndHypervisorType(List zoneIds, HypervisorType hypervisorType, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index c63af0a237ba..9b6f0f7ac3a6 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -20,6 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.cpu.CPU; @@ -38,6 +40,7 @@ import com.cloud.org.Grouping; import com.cloud.org.Managed; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; @@ -413,4 +416,19 @@ public List listEnabledClusterIdsByZoneHypervisorArch(Long zoneId, Hypervi } return customSearch(sc, null); } + + @Override + public List listByZonesAndHypervisorType(List zoneIds, HypervisorType hypervisorType, Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + sc.setParameters("hypervisorType", hypervisorType.toString()); + return listBy(sc, filter); + } } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java index 6af16bbace99..d14ccbe86ca6 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java @@ -24,5 +24,5 @@ public interface DomainVlanMapDao extends GenericDao { public List listDomainVlanMapsByDomain(long domainId); public List listDomainVlanMapsByVlan(long vlanDbId); - public DomainVlanMapVO findDomainVlanMap(long domainId, long vlanDbId); + public DomainVlanMapVO findDomainVlanMap(Long domainId, long vlanDbId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java index f789721d5fd6..0b4c781349fd 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java @@ -46,9 +46,9 @@ public List listDomainVlanMapsByVlan(long vlanDbId) { } @Override - public DomainVlanMapVO findDomainVlanMap(long domainId, long vlanDbId) { + public DomainVlanMapVO findDomainVlanMap(Long domainId, long vlanDbId) { SearchCriteria sc = DomainVlanSearch.create(); - sc.setParameters("domainId", domainId); + sc.setParametersIfNotNull("domainId", domainId); sc.setParameters("vlanDbId", vlanDbId); return findOneIncludingRemovedBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 090b019334f4..5f5b2affee08 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -218,7 +218,7 @@ int countHostsByMsResourceStateTypeAndHypervisorType(long msId, List listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 2d8fcca6cdb7..99c9a979c3bf 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -1520,7 +1520,7 @@ private List findHostIdsByHostTags(String hostTags){ } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { @@ -1534,7 +1534,7 @@ public List findHostsWithTagRuleThatMatchComputeOferringTags(String comp public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 7a00829fd44e..0d86ca0e48c7 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao { HostTagResponse newHostTagResponse(HostTagVO hostTag); List searchByIds(Long... hostTagIds); + + /** + * List all host tags defined on hosts within a cluster + */ + List listByClusterId(Long clusterId); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index 4aa14a31cfcf..d3fee6a26761 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase implements private final SearchBuilder stSearch; private final SearchBuilder tagIdsearch; private final SearchBuilder ImplicitTagsSearch; + private final GenericSearchBuilder tagSearch; @Inject private ConfigurationDao _configDao; + @Inject + private HostDao hostDao; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); @@ -72,6 +76,11 @@ public HostTagsDaoImpl() { ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ); ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); ImplicitTagsSearch.done(); + + tagSearch = createSearchBuilder(String.class); + tagSearch.selectFields(tagSearch.entity().getTag()); + tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN); + tagSearch.done(); } @Override @@ -235,4 +244,15 @@ public List searchByIds(Long... tagIds) { return tagList; } + + @Override + public List listByClusterId(Long clusterId) { + List hostIds = hostDao.listIdsByClusterId(clusterId); + if (CollectionUtils.isEmpty(hostIds)) { + return new ArrayList<>(); + } + SearchCriteria sc = tagSearch.create(); + sc.setParameters("hostIdIN", hostIds.toArray()); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index fdca6e43f00f..8406f80ed09d 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -24,6 +24,7 @@ import com.cloud.network.Network.GuestType; import com.cloud.network.Network.State; import com.cloud.network.Networks.TrafficType; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.fsm.StateDao; @@ -96,8 +97,13 @@ public interface NetworkDao extends GenericDao, StateDao serviceProviderMap); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType, Filter filter); + List listByZoneAndTrafficType(long zoneId, TrafficType trafficType); + List listByZonesTrafficTypeAndOwners(List zoneIds, final TrafficType trafficType, + List accountIds, List domainIds, Filter filter); + void setCheckForGc(long networkId); int getNetworkCountByNetworkOffId(long networkOfferingId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 9f7ffabac930..f3ece4879040 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -19,6 +19,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -29,9 +30,9 @@ import javax.inject.Inject; import javax.persistence.TableGenerator; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.network.Network; @@ -63,6 +64,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.SequenceFetcher; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @Component @@ -632,12 +634,46 @@ public List listBy(final long accountId, final long dataCenterId, fin } @Override - public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType, Filter filter) { final SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("datacenter", zoneId); sc.setParameters("trafficType", trafficType); - return listBy(sc, null); + return listBy(sc, filter); + } + + @Override + public List listByZoneAndTrafficType(final long zoneId, final TrafficType trafficType) { + return listByZoneAndTrafficType(zoneId, trafficType, null); + } + + @Override + public List listByZonesTrafficTypeAndOwners(List zoneIds, final TrafficType trafficType, + List accountIds, List domainIds, Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + sb.and("trafficType", sb.entity().getTrafficType(), Op.EQ); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + sc.setParameters("trafficType", trafficType); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + return listBy(sc, filter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java index 02abaacd854e..f2572ba91c21 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java @@ -203,9 +203,13 @@ public class NetworkVO implements Network { @Column(name = "private_mtu") Integer privateMtu; + @Column(name = "keep_mac_address_on_public_nic") + private boolean keepMacAddressOnPublicNic = true; + @Transient Integer networkCidrSize; + public NetworkVO() { uuid = UUID.randomUUID().toString(); } @@ -773,4 +777,13 @@ public Integer getNetworkCidrSize() { public void setNetworkCidrSize(Integer networkCidrSize) { this.networkCidrSize = networkCidrSize; } + + @Override + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + + public void setKeepMacAddressOnPublicNic(boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java index ccba6bb18893..606bdaaaa7a7 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java @@ -19,9 +19,21 @@ import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.utils.db.GenericDao; +import java.util.Date; +import java.util.List; + public interface PublicIpQuarantineDao extends GenericDao { PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId); PublicIpQuarantineVO findByIpAddress(String publicIpAddress); + + /** + * Returns a list of public IP addresses that are actively quarantined at the specified date and the previous owner differs from the specified user. + * + * @param userId used to check against the IP's previous owner; + * @param date used to check if the quarantine is active; + * @return a list of PublicIpQuarantineVOs. + */ + List listQuarantinedIpAddressesToUser(Long userId, Date date); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java index a1b789b8a46b..0c47a0d36e30 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java @@ -26,6 +26,8 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import java.util.Date; +import java.util.List; @Component public class PublicIpQuarantineDaoImpl extends GenericDaoBase implements PublicIpQuarantineDao { @@ -33,6 +35,8 @@ public class PublicIpQuarantineDaoImpl extends GenericDaoBase ipAddressSearchBuilder; + private SearchBuilder quarantinedIpAddressesSearch; + @Inject IPAddressDao ipAddressDao; @@ -47,8 +51,16 @@ public void init() { publicIpAddressByIdSearch.join("quarantineJoin", ipAddressSearchBuilder, ipAddressSearchBuilder.entity().getId(), publicIpAddressByIdSearch.entity().getPublicIpAddressId(), JoinBuilder.JoinType.INNER); + quarantinedIpAddressesSearch = createSearchBuilder(); + quarantinedIpAddressesSearch.and("previousOwnerId", quarantinedIpAddressesSearch.entity().getPreviousOwnerId(), SearchCriteria.Op.NEQ); + quarantinedIpAddressesSearch.and(); + quarantinedIpAddressesSearch.op("removedIsNull", quarantinedIpAddressesSearch.entity().getRemoved(), SearchCriteria.Op.NULL); + quarantinedIpAddressesSearch.and("endDate", quarantinedIpAddressesSearch.entity().getEndDate(), SearchCriteria.Op.GT); + quarantinedIpAddressesSearch.cp(); + ipAddressSearchBuilder.done(); publicIpAddressByIdSearch.done(); + quarantinedIpAddressesSearch.done(); } @Override @@ -68,4 +80,14 @@ public PublicIpQuarantineVO findByIpAddress(String publicIpAddress) { return findOneBy(sc, filter); } + + @Override + public List listQuarantinedIpAddressesToUser(Long userId, Date date) { + SearchCriteria sc = quarantinedIpAddressesSearch.create(); + + sc.setParameters("previousOwnerId", userId); + sc.setParameters("endDate", date); + + return searchIncludingRemoved(sc, null, false, false); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java index 793334731095..2b18bcbd87bd 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDao.java @@ -31,4 +31,6 @@ public interface SecurityGroupDao extends GenericDao { List findByAccountAndNames(Long accountId, String... names); int removeByAccountId(long accountId); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java index 019cf5fec462..ffb45629b625 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/security/dao/SecurityGroupDaoImpl.java @@ -129,4 +129,14 @@ public boolean expunge(Long id) { txn.commit(); return result; } + + @Override + public List listByIds(List ids) { + SearchBuilder idsSearch = createSearchBuilder(); + idsSearch.and("ids", idsSearch.entity().getId(), SearchCriteria.Op.IN); + idsSearch.done(); + SearchCriteria sc = idsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java index e942eadb8ffb..742d3f2f82ee 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java @@ -108,6 +108,9 @@ public class VpcVO implements Vpc { @Column(name = "use_router_ip_resolver") boolean useRouterIpResolver = false; + @Column(name = "keep_mac_address_on_public_nic") + private boolean keepMacAddressOnPublicNic = true; + @Transient boolean rollingRestart = false; @@ -321,4 +324,13 @@ public boolean useRouterIpAsResolver() { public void setUseRouterIpResolver(boolean useRouterIpResolver) { this.useRouterIpResolver = useRouterIpResolver; } + + @Override + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + + public void setKeepMacAddressOnPublicNic(boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java index 9beea0037444..ea97f663768f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java @@ -30,7 +30,7 @@ public interface DiskOfferingDao extends GenericDao { List listAllBySizeAndProvisioningType(long size, Storage.ProvisioningType provisioningType); - List findCustomDiskOfferings(); + List listCustomDiskOfferings(); List listByStorageTag(String tag); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java index 4ca3fe9f12ac..44e6551da17d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage.dao; +import static org.apache.cloudstack.query.QueryService.SortKeyAscending; + import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -33,6 +35,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.utils.db.Attribute; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -138,13 +141,14 @@ public List listAllBySizeAndProvisioningType(long size, Storage. } @Override - public List findCustomDiskOfferings() { + public List listCustomDiskOfferings() { SearchBuilder sb = createSearchBuilder(); sb.and("customized", sb.entity().isCustomized(), SearchCriteria.Op.EQ); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("customized", true); - return listBy(sc); + Filter searchFilter = new Filter(DiskOfferingVO.class, "sortKey", SortKeyAscending.value()); + return listBy(sc, searchFilter); } @Override diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java index 80e1b7d4d4be..f167b5731878 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -170,6 +170,7 @@ protected void init() { CountSnapshotsByAccount.select(null, Func.COUNT, null); CountSnapshotsByAccount.and("account", CountSnapshotsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); CountSnapshotsByAccount.and("status", CountSnapshotsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountSnapshotsByAccount.and("snapshotTypeNEQ", CountSnapshotsByAccount.entity().getSnapshotType(), SearchCriteria.Op.NIN); CountSnapshotsByAccount.and("removed", CountSnapshotsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CountSnapshotsByAccount.done(); @@ -220,6 +221,7 @@ public Long countSnapshotsForAccount(long accountId) { SearchCriteria sc = CountSnapshotsByAccount.create(); sc.setParameters("account", accountId); sc.setParameters("status", State.Error, State.Destroyed); + sc.setParameters("snapshotTypeNEQ", Snapshot.Type.GROUP.ordinal()); return customSearch(sc, null).get(0); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 4c9f906b68a9..aec06d6d0003 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -106,4 +106,6 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findActiveSystemTemplateByHypervisorArchAndUrlPath(HypervisorType hypervisorType, CPU.CPUArch arch, String urlPathSuffix); + + VMTemplateVO findByAccountAndName(Long accountId, String templateName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 9b5d0edc599d..8c6e3fe0983f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -945,4 +945,12 @@ public boolean updateState( } return rows > 0; } + + @Override + public VMTemplateVO findByAccountAndName(Long accountId, String templateName) { + SearchCriteria sc = NameAccountIdSearch.create(); + sc.setParameters("name", templateName); + sc.setParameters("accountId", accountId); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 717e3e782f2f..7d566a1ea01e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -46,6 +46,8 @@ public interface VolumeDao extends GenericDao, StateDao findByInstanceAndType(long id, Volume.Type vType); + List findByInstanceAndNotStates(long id, Volume.State...states); + List findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index fce4d1f7233d..91f1c7f5eb69 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -208,6 +208,17 @@ public List findByInstanceAndType(long id, Type vType) { return listBy(sc); } + @Override + public List findByInstanceAndNotStates(long id, Volume.State...states) { + SearchBuilder sb = createSearchBuilder(); + sb.and("instanceId", sb.entity().getInstanceId(), Op.EQ); + sb.and("state", sb.entity().getState(), Op.NIN); + SearchCriteria sc = sb.create(); + sc.setParameters("instanceId", id); + sc.setParameters("state", (Object[]) states); + return listBy(sc); + } + @Override public List findIncludingRemovedByInstanceAndType(long id, Type vType) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java index bacb09b98793..5efaea40a943 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagDao.java @@ -20,11 +20,13 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.response.ResourceTagResponse; + import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import org.apache.cloudstack.api.response.ResourceTagResponse; public interface ResourceTagDao extends GenericDao { @@ -60,4 +62,13 @@ public interface ResourceTagDao extends GenericDao { void removeByResourceIdAndKey(long resourceId, ResourceObjectType resourceType, String key); List listByResourceUuid(String resourceUuid); + + List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter); + + ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, String value); + + List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, String key); + } diff --git a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java index cc9d99e6ab16..22c7b7b2ee5b 100644 --- a/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/tags/dao/ResourceTagsDaoImpl.java @@ -16,19 +16,22 @@ // under the License. package com.cloud.tags.dao; -import java.util.List; -import java.util.Set; -import java.util.Map; import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.ResourceTagVO; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; @@ -120,4 +123,62 @@ public List listByResourceUuid(String resourceUuid) { sc.setParameters("resourceUuid", resourceUuid); return listBy(sc); } + + @Override + public List listByResourceTypeKeyPrefixAndOwners(ResourceObjectType resourceType, String key, + List accountIds, List domainIds, + Filter filter) { + GenericSearchBuilder sb = createSearchBuilder(String.class); + sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getValue()); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key + "%"); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + return customSearch(sc, filter); + } + + @Override + public ResourceTagVO findByResourceTypeKeyPrefixAndValue(ResourceObjectType resourceType, String key, + String value) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.and("value", sb.entity().getValue(), Op.EQ); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("key", key + "%"); + sc.setParameters("value", value); + return findOneBy(sc); + } + + @Override + public List listByResourceTypeIdAndKeyPrefix(ResourceObjectType resourceType, long resourceId, + String key) { + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceType", sb.entity().getResourceType(), Op.EQ); + sb.and("resourceId", sb.entity().getResourceId(), Op.EQ); + sb.and("key", sb.entity().getKey(), Op.LIKE); + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("resourceType", resourceType); + sc.setParameters("resourceId", resourceId); + sc.setParameters("key", key + "%"); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java index 445a59310fb4..377b2f913755 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.upgrade; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -96,7 +97,9 @@ public DbUpgrade[] getPath(final CloudStackVersion fromVersion, final CloudStack // we cannot find the version specified, so get the // most recent one immediately before this version if (!contains(fromVersion)) { - return getPath(getRecentVersion(fromVersion), toVersion); + DbUpgrade[] dbUpgrades = getPath(getRecentVersion(fromVersion), toVersion); + return Arrays.stream(dbUpgrades).filter(up -> CloudStackVersion.compare(up.getUpgradedVersion(), fromVersion.toString()) > 0) + .toArray(DbUpgrade[]::new); } final Predicate predicate; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 292bafefbb65..2acb4138d234 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -1073,7 +1073,7 @@ protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplate } Hypervisor.HypervisorType hypervisorType = templateDetails.getHypervisorType(); updateSystemVMEntries(templateId, hypervisorType); - updateConfigurationParams(hypervisorType, templateDetails.getName(), zoneId); + updateConfigurationParams(hypervisorType, templateVO.getName(), zoneId); } protected void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java index 68100e164018..e0aa38a717b2 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -57,8 +57,4 @@ public void performDataMigration(Connection conn) { public InputStream[] getCleanupScripts() { return null; } - - @Override - public void updateSystemVmTemplates(Connection conn) { - } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java index df4743894c9d..393f20399503 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java @@ -17,7 +17,12 @@ package com.cloud.upgrade.dao; import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @@ -42,4 +47,46 @@ public InputStream[] getPrepareScripts() { return new InputStream[] {script}; } + + @Override + public void performDataMigration(Connection conn) { + unhideJsInterpretationEnabled(conn); + } + + protected void unhideJsInterpretationEnabled(Connection conn) { + String value = getJsInterpretationEnabled(conn); + if (value != null) { + updateJsInterpretationEnabledFields(conn, value); + } + } + + protected String getJsInterpretationEnabled(Connection conn) { + String query = "SELECT value FROM cloud.configuration WHERE name = 'js.interpretation.enabled' AND category = 'Hidden';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getString("value"); + } + logger.debug("Unable to retrieve value of hidden configuration 'js.interpretation.enabled'. The configuration may already be unhidden."); + return null; + } catch (SQLException e) { + throw new CloudRuntimeException("Error while retrieving value of hidden configuration 'js.interpretation.enabled'.", e); + } + } + + protected void updateJsInterpretationEnabledFields(Connection conn, String encryptedValue) { + String query = "UPDATE cloud.configuration SET value = ?, category = 'System', component = 'JsInterpreter', is_dynamic = 1 WHERE name = 'js.interpretation.enabled';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + String decryptedValue = DBEncryptionUtil.decrypt(encryptedValue); + logger.info("Updating setting 'js.interpretation.enabled' to decrypted value [{}], category 'System', component 'JsInterpreter', and is_dynamic '1'.", decryptedValue); + pstmt.setString(1, decryptedValue); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Error while unhiding configuration 'js.interpretation.enabled'.", e); + } catch (CloudRuntimeException e) { + logger.warn("Error while decrypting configuration 'js.interpretation.enabled'. The configuration may already be decrypted."); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 96a1b6e3bd16..7767b7512b91 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -26,6 +26,8 @@ public interface NicDao extends GenericDao { List listByVmId(long instanceId); + int countByVmId(long instanceId); + List listByVmIdOrderByDeviceId(long instanceId); List listIpAddressInNetwork(long networkConfigId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index 78966a09e97c..6202dabf730e 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -126,6 +126,13 @@ public List listByVmId(long instanceId) { return listBy(sc); } + @Override + public int countByVmId(long instanceId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("instance", instanceId); + return getCount(sc); + } + @Override public List listByVmIdOrderByDeviceId(long instanceId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 23541c2431e7..1a5b8cedd9ea 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.Pair; @@ -192,4 +193,10 @@ List searchRemovedByRemoveDate(final Date startDate, final Date en int getVmCountByOfferingNotInDomain(Long serviceOfferingId, List domainIds); List listByIdsIncludingRemoved(List ids); + + List listDeleteProtectedVmsByAccountId(long accountId); + + List listDeleteProtectedVmsByDomainIds(Set domainIds); + + List listIdsByHostIdForVolumeStats(long hostIds); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 589a63ea0d84..ae1e838649ba 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -25,11 +25,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -106,6 +108,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder IdsPowerStateSelectSearch; GenericSearchBuilder CountByOfferingId; GenericSearchBuilder CountUserVmNotInDomain; + SearchBuilder DeleteProtectedVmSearchByAccount; + SearchBuilder DeleteProtectedVmSearchByDomainIds; @Inject ResourceTagDao tagsDao; @@ -354,7 +358,8 @@ protected void init() { IdsPowerStateSelectSearch.entity().getPowerHostId(), IdsPowerStateSelectSearch.entity().getPowerState(), IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(), - IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime()); + IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime(), + IdsPowerStateSelectSearch.entity().getState()); IdsPowerStateSelectSearch.done(); CountByOfferingId = createSearchBuilder(Integer.class); @@ -368,6 +373,19 @@ protected void init() { CountUserVmNotInDomain.and("domainIdsNotIn", CountUserVmNotInDomain.entity().getDomainId(), Op.NIN); CountUserVmNotInDomain.done(); + DeleteProtectedVmSearchByAccount = createSearchBuilder(); + DeleteProtectedVmSearchByAccount.selectFields(DeleteProtectedVmSearchByAccount.entity().getUuid()); + DeleteProtectedVmSearchByAccount.and(ApiConstants.ACCOUNT_ID, DeleteProtectedVmSearchByAccount.entity().getAccountId(), Op.EQ); + DeleteProtectedVmSearchByAccount.and(ApiConstants.DELETE_PROTECTION, DeleteProtectedVmSearchByAccount.entity().isDeleteProtection(), Op.EQ); + DeleteProtectedVmSearchByAccount.and(ApiConstants.REMOVED, DeleteProtectedVmSearchByAccount.entity().getRemoved(), Op.NULL); + DeleteProtectedVmSearchByAccount.done(); + + DeleteProtectedVmSearchByDomainIds = createSearchBuilder(); + DeleteProtectedVmSearchByDomainIds.selectFields(DeleteProtectedVmSearchByDomainIds.entity().getUuid()); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.DOMAIN_IDS, DeleteProtectedVmSearchByDomainIds.entity().getDomainId(), Op.IN); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.DELETE_PROTECTION, DeleteProtectedVmSearchByDomainIds.entity().isDeleteProtection(), Op.EQ); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.REMOVED, DeleteProtectedVmSearchByDomainIds.entity().getRemoved(), Op.NULL); + DeleteProtectedVmSearchByDomainIds.done(); } @Override @@ -1088,10 +1106,14 @@ public Map updatePowerState( private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) { State instanceState = instance.getState(); + if (instanceState == null) { + logger.warn("VM {} has null instance state during power state sync check, treating as out of sync", instance); + return false; + } if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running) || (powerState == VirtualMachine.PowerState.PowerOn && instanceState == State.Stopped)) { HostVO instanceHost = hostDao.findById(instance.getHostId()); - HostVO powerHost = powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); + HostVO powerHost = instance.getHostId() != null && powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); logger.debug("VM: {} on host: {} and power host : {} is in {} state, but power state is {}", instance, instanceHost, powerHost, instanceState, powerState); return false; @@ -1296,4 +1318,38 @@ public List listByIdsIncludingRemoved(List ids) { sc.setParameters("ids", ids.toArray()); return listIncludingRemovedBy(sc); } + + @Override + public List listDeleteProtectedVmsByAccountId(long accountId) { + SearchCriteria sc = DeleteProtectedVmSearchByAccount.create(); + sc.setParameters(ApiConstants.ACCOUNT_ID, accountId); + sc.setParameters(ApiConstants.DELETE_PROTECTION, true); + Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); + return listBy(sc, filter); + } + + @Override + public List listDeleteProtectedVmsByDomainIds(Set domainIds) { + SearchCriteria sc = DeleteProtectedVmSearchByDomainIds.create(); + sc.setParameters(ApiConstants.DOMAIN_IDS, domainIds.toArray()); + sc.setParameters(ApiConstants.DELETE_PROTECTION, true); + Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); + return listBy(sc, filter); + } + + @Override + public List listIdsByHostIdForVolumeStats(long hostId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.selectFields(sb.entity().getId()); + sb.and().op("host", sb.entity().getHostId(), SearchCriteria.Op.EQ); + sb.or().op("hostNull", sb.entity().getHostId(), Op.NULL); + sb.and("lastHost", sb.entity().getLastHostId(), SearchCriteria.Op.EQ); + sb.cp(); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("host", hostId); + sc.setParameters("lastHost", hostId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 0f8a10fb7be6..d589f9e6bef8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -103,6 +103,18 @@ public class BackupVO implements Backup { @Column(name = "backup_schedule_id") private Long backupScheduleId; + @Column(name = "from_checkpoint_id") + private String fromCheckpointId; + + @Column(name = "to_checkpoint_id") + private String toCheckpointId; + + @Column(name = "checkpoint_create_time") + private Long checkpointCreateTime; + + @Column(name = "host_id") + private Long hostId; + @Transient Map details; @@ -288,4 +300,40 @@ public Long getBackupScheduleId() { public void setBackupScheduleId(Long backupScheduleId) { this.backupScheduleId = backupScheduleId; } + + @Override + public String getFromCheckpointId() { + return fromCheckpointId; + } + + public void setFromCheckpointId(String fromCheckpointId) { + this.fromCheckpointId = fromCheckpointId; + } + + @Override + public String getToCheckpointId() { + return toCheckpointId; + } + + public void setToCheckpointId(String toCheckpointId) { + this.toCheckpointId = toCheckpointId; + } + + @Override + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + + public void setCheckpointCreateTime(Long checkpointCreateTime) { + this.checkpointCreateTime = checkpointCreateTime; + } + + @Override + public Long getHostId() { + return hostId; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java new file mode 100644 index 000000000000..ec9b927b63e4 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/ImageTransferVO.java @@ -0,0 +1,242 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name = "image_transfer") +public class ImageTransferVO implements ImageTransfer { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "backup_id") + private Long backupId; + + @Column(name = "volume_id") + private long volumeId; + + @Column(name = "host_id") + private long hostId; + + @Column(name = "socket") + private String socket; + + @Column(name = "file") + private String file; + + @Column(name = "transfer_url") + private String transferUrl; + + @Enumerated(value = EnumType.STRING) + @Column(name = "phase") + private Phase phase; + + @Enumerated(value = EnumType.STRING) + @Column(name = "direction") + private Direction direction; + + @Enumerated(value = EnumType.STRING) + @Column(name = "backend") + private Backend backend; + + @Column(name = "signed_ticket_id") + private String signedTicketId; + + @Column(name = "account_id") + Long accountId; + + @Column(name = "domain_id") + Long domainId; + + @Column(name = "data_center_id") + Long dataCenterId; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = "updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date updated; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public ImageTransferVO() { + } + + private ImageTransferVO(String uuid, long volumeId, long hostId, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this.uuid = uuid; + this.volumeId = volumeId; + this.hostId = hostId; + this.phase = phase; + this.direction = direction; + this.accountId = accountId; + this.domainId = domainId; + this.dataCenterId = dataCenterId; + this.created = new Date(); + } + + public ImageTransferVO(String uuid, Long backupId, long volumeId, long hostId, String socket, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.backupId = backupId; + this.socket = socket; + this.backend = Backend.nbd; + } + + public ImageTransferVO(String uuid, long volumeId, long hostId, String file, Phase phase, Direction direction, Long accountId, Long domainId, Long dataCenterId) { + this(uuid, volumeId, hostId, phase, direction, accountId, domainId, dataCenterId); + this.file = file; + this.backend = Backend.file; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public Long getBackupId() { + return backupId; + } + + public void setBackupId(long backupId) { + this.backupId = backupId; + } + + @Override + public long getVolumeId() { + return volumeId; + } + + public void setVolumeId(long volumeId) { + this.volumeId = volumeId; + } + + @Override + public long getHostId() { + return hostId; + } + + public void setHostId(long hostId) { + this.hostId = hostId; + } + + public void setSocket(String socket) { + this.socket = socket; + } + + @Override + public String getTransferUrl() { + return transferUrl; + } + + public void setTransferUrl(String transferUrl) { + this.transferUrl = transferUrl; + } + + @Override + public Phase getPhase() { + return phase; + } + + public void setPhase(Phase phase) { + this.phase = phase; + this.updated = new Date(); + } + + @Override + public Direction getDirection() { + return direction; + } + + public void setDirection(Direction direction) { + this.direction = direction; + } + + @Override + public Backend getBackend() { + return backend; + } + + @Override + public String getSignedTicketId() { + return signedTicketId; + } + + public void setSignedTicketId(String signedTicketId) { + this.signedTicketId = signedTicketId; + } + + @Override + public Class getEntityType() { + return ImageTransfer.class; + } + + @Override + public String getName() { + return null; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public long getDataCenterId() { + return dataCenterId; + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java index ee1783a9c896..87b7dab1ff7b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java @@ -21,20 +21,14 @@ import java.util.List; import com.cloud.utils.DateUtil; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.BackupScheduleVO; import com.cloud.utils.db.GenericDao; public interface BackupScheduleDao extends GenericDao { - BackupScheduleVO findByVM(Long vmId); - List listByVM(Long vmId); BackupScheduleVO findByVMAndIntervalType(Long vmId, DateUtil.IntervalType intervalType); List getSchedulesToExecute(Date currentTimestamp); - - BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index d9cf7b63680b..972af73391af 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -17,28 +17,23 @@ package org.apache.cloudstack.backup.dao; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.Date; import java.util.List; import javax.annotation.PostConstruct; -import javax.inject.Inject; import com.cloud.utils.DateUtil; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.backup.BackupSchedule; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.TransactionLegacy; import org.apache.cloudstack.backup.BackupScheduleVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.dao.VMInstanceDao; public class BackupScheduleDaoImpl extends GenericDaoBase implements BackupScheduleDao { - - @Inject - VMInstanceDao vmInstanceDao; - private SearchBuilder backupScheduleSearch; private SearchBuilder executableSchedulesSearch; @@ -59,13 +54,6 @@ protected void init() { executableSchedulesSearch.done(); } - @Override - public BackupScheduleVO findByVM(Long vmId) { - SearchCriteria sc = backupScheduleSearch.create(); - sc.setParameters("vm_id", vmId); - return findOneBy(sc); - } - @Override public List listByVM(Long vmId) { SearchCriteria sc = backupScheduleSearch.create(); @@ -88,21 +76,19 @@ public List getSchedulesToExecute(Date currentTimestamp) { return listBy(sc); } + @DB @Override - public BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule) { - VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(schedule.getVmId()); - BackupScheduleResponse response = new BackupScheduleResponse(); - response.setId(schedule.getUuid()); - response.setVmId(vm.getUuid()); - response.setVmName(vm.getHostName()); - response.setIntervalType(schedule.getScheduleType()); - response.setSchedule(schedule.getSchedule()); - response.setTimezone(schedule.getTimezone()); - response.setMaxBackups(schedule.getMaxBackups()); - if (schedule.getQuiesceVM() != null) { - response.setQuiesceVM(schedule.getQuiesceVM()); + public boolean remove(Long id) { + String sql = "UPDATE backups SET backup_schedule_id = NULL WHERE backup_schedule_id = ?"; + TransactionLegacy transaction = TransactionLegacy.currentTxn(); + try { + PreparedStatement preparedStatement = transaction.prepareAutoCloseStatement(sql); + preparedStatement.setLong(1, id); + preparedStatement.executeUpdate(); + return super.remove(id); + } catch (SQLException e) { + logger.warn("Unable to clean up backup schedules references from the backups table.", e); + return false; } - response.setObjectName("backupschedule"); - return response; } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java new file mode 100644 index 000000000000..9a9fbeb8814e --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDao.java @@ -0,0 +1,36 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.ImageTransferVO; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDao; + +public interface ImageTransferDao extends GenericDao { + List listByBackupId(Long backupId); + ImageTransferVO findByUuid(String uuid); + ImageTransferVO findByVolume(Long volumeId); + ImageTransferVO findUnfinishedByVolume(Long volumeId); + List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction); + List listByZonesAndOwners(List zoneIds, List accountIds, List domainIds, + Filter filter); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java new file mode 100644 index 000000000000..2e36f924e4ef --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/ImageTransferDaoImpl.java @@ -0,0 +1,136 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.Collections; +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.backup.ImageTransferVO; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class ImageTransferDaoImpl extends GenericDaoBase implements ImageTransferDao { + + private SearchBuilder backupIdSearch; + private SearchBuilder uuidSearch; + private SearchBuilder volumeSearch; + private SearchBuilder volumeUnfinishedSearch; + private SearchBuilder phaseDirectionSearch; + + public ImageTransferDaoImpl() { + } + + @PostConstruct + protected void init() { + backupIdSearch = createSearchBuilder(); + backupIdSearch.and("backupId", backupIdSearch.entity().getBackupId(), SearchCriteria.Op.EQ); + backupIdSearch.done(); + + uuidSearch = createSearchBuilder(); + uuidSearch.and("uuid", uuidSearch.entity().getUuid(), SearchCriteria.Op.EQ); + uuidSearch.done(); + + volumeSearch = createSearchBuilder(); + volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeSearch.done(); + + volumeUnfinishedSearch = createSearchBuilder(); + volumeUnfinishedSearch.and("volumeId", volumeUnfinishedSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeUnfinishedSearch.and("phase", volumeUnfinishedSearch.entity().getPhase(), SearchCriteria.Op.NEQ); + volumeUnfinishedSearch.done(); + + phaseDirectionSearch = createSearchBuilder(); + phaseDirectionSearch.and("phase", phaseDirectionSearch.entity().getPhase(), SearchCriteria.Op.EQ); + phaseDirectionSearch.and("direction", phaseDirectionSearch.entity().getDirection(), SearchCriteria.Op.EQ); + phaseDirectionSearch.done(); + } + + @Override + public List listByBackupId(Long backupId) { + SearchCriteria sc = backupIdSearch.create(); + sc.setParameters("backupId", backupId); + return listBy(sc); + } + + @Override + public ImageTransferVO findByUuid(String uuid) { + SearchCriteria sc = uuidSearch.create(); + sc.setParameters("uuid", uuid); + return findOneBy(sc); + } + + @Override + public ImageTransferVO findByVolume(Long volumeId) { + SearchCriteria sc = volumeSearch.create(); + sc.setParameters("volumeId", volumeId); + return findOneBy(sc); + } + + @Override + public ImageTransferVO findUnfinishedByVolume(Long volumeId) { + SearchCriteria sc = volumeUnfinishedSearch.create(); + sc.setParameters("volumeId", volumeId); + sc.setParameters("phase", ImageTransferVO.Phase.finished.toString()); + return findOneBy(sc); + } + + @Override + public List listByPhaseAndDirection(ImageTransfer.Phase phase, ImageTransfer.Direction direction) { + SearchCriteria sc = phaseDirectionSearch.create(); + sc.setParameters("phase", phase); + sc.setParameters("direction", direction); + return listBy(sc); + } + + @Override + public List listByZonesAndOwners(List zoneIds, List accountIds, List domainIds, + Filter filter) { + if (CollectionUtils.isEmpty(zoneIds)) { + return Collections.emptyList(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.IN); + boolean accountIdsNotEmpty = CollectionUtils.isNotEmpty(accountIds); + boolean domainIdsNotEmpty = CollectionUtils.isNotEmpty(domainIds); + if (accountIdsNotEmpty || domainIdsNotEmpty) { + sb.and().op("account", sb.entity().getAccountId(), SearchCriteria.Op.IN); + sb.or("domain", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.done(); + final SearchCriteria sc = sb.create(); + sc.setParameters("dataCenterId", zoneIds.toArray()); + if (accountIdsNotEmpty) { + sc.setParameters("account", accountIds.toArray()); + } + if (domainIdsNotEmpty) { + sc.setParameters("domain", domainIds.toArray()); + } + + return listBy(sc, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 6aeee1ad1ccd..3329983d711e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -51,6 +51,8 @@ public interface SnapshotDataStoreDao extends GenericDao listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State... state); + List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId); SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index bbb2b4f3a88e..8b7a2b78de7e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -68,6 +68,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; private SearchBuilder idStateNinSearch; + private SearchBuilder idEqRoleEqStateInSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; @@ -151,6 +152,11 @@ public boolean configure(String name, Map params) throws Configu idStateNinSearch.and(STATE, idStateNinSearch.entity().getState(), SearchCriteria.Op.NOTIN); idStateNinSearch.done(); + idEqRoleEqStateInSearch = createSearchBuilder(); + idEqRoleEqStateInSearch.and(SNAPSHOT_ID, idEqRoleEqStateInSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + idEqRoleEqStateInSearch.and(STORE_ROLE, idEqRoleEqStateInSearch.entity().getRole(), SearchCriteria.Op.EQ); + idEqRoleEqStateInSearch.and(STATE, idEqRoleEqStateInSearch.entity().getState(), SearchCriteria.Op.IN); + snapshotVOSearch = snapshotDao.createSearchBuilder(); snapshotVOSearch.and(VOLUME_ID, snapshotVOSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); snapshotVOSearch.done(); @@ -387,6 +393,15 @@ public SnapshotDataStoreVO findBySnapshotIdAndDataStoreRoleAndState(long snapsho return findOneBy(sc); } + @Override + public List listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, State... state) { + SearchCriteria sc = idEqRoleEqStateInSearch.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STORE_ROLE, role); + sc.setParameters(STATE, (Object[])state); + return listBy(sc); + } + @Override public SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId) { try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) { diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java index 4735202a7623..82ba9445b25c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDao.java @@ -29,4 +29,6 @@ public interface SharedFSDao extends GenericDao, StateDao listSharedFSToBeDestroyed(Date date); SharedFSVO findSharedFSByNameAccountDomain(String name, Long accountId, Long domainId); + + SharedFSVO findByVm(long vmId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java index da6220716715..dd23787f982f 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/sharedfs/dao/SharedFSDaoImpl.java @@ -114,4 +114,14 @@ public SharedFSVO findSharedFSByNameAccountDomain(String name, Long accountId, L sc.setParameters("domainId", domainId); return findOneBy(sc); } + + @Override + public SharedFSVO findByVm(long vmId) { + SearchBuilder sb = createSearchBuilder(); + sb.and("vmId", sb.entity().getVmId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("vmId", vmId); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml index 1846c3c62a0e..2c6869bd81e3 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml @@ -73,5 +73,7 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index edc14d9fa0cc..26181d3fce0d 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -116,6 +116,7 @@ + @@ -236,13 +237,11 @@ - - @@ -273,6 +272,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql index 3dd6c18f57c5..4405250f2711 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql @@ -31,7 +31,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); -- Create a new group for Usage Server related configurations -INSERT INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); +INSERT IGNORE INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); UPDATE `cloud`.`configuration_subgroup` set `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server'), `precedence` = 1 WHERE `name`='Usage'; UPDATE `cloud`.`configuration` SET `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server') where `subgroup_id` = (SELECT `id` FROM `cloud`.`configuration_subgroup` WHERE `name` = 'Usage'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4c65f37e0fe0..000b54b72078 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -687,22 +687,23 @@ CREATE TABLE IF NOT EXISTS `cloud`.`backup_details` ( UPDATE `cloud`.`backups` b INNER JOIN `cloud`.`vm_instance` vm ON b.vm_id = vm.id SET b.backed_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id @@ -711,22 +712,23 @@ SET b.backed_volumes = ( -- Add diskOfferingId, deviceId, minIops and maxIops to backup_volumes in vm_instance table UPDATE `cloud`.`vm_instance` vm SET vm.backup_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 858c46a7c1ee..fbb2fd079f9c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -19,7 +19,6 @@ -- Schema upgrade from 4.21.0.0 to 4.22.0.0 --; - -- health check status as enum CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 54baf226ac43..2f104568c140 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,3 +18,9 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; + +-- Entries remaining on `cloud`.`resource_reservation` during the upgrade process are stale, so delete them. +-- This script was added to normalize volume/primary storage reservations that got stuck due to a bug on VM deployment, +-- but it is more interesting to introduce a smarter logic to clean these stale reservations in the future without the need +-- for upgrades (for instance, by having a heartbeat_time column for the reservations and automatically cleaning old entries). +DELETE FROM `cloud`.`resource_reservation`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 369159e48360..baa20e9f0ad5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -34,7 +34,28 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; -- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; +-- Sanitize legacy network-level addressing fields for Public networks +UPDATE `cloud`.`networks` +SET `broadcast_uri` = NULL, + `gateway` = NULL, + `cidr` = NULL, + `ip6_gateway` = NULL, + `ip6_cidr` = NULL +WHERE `traffic_type` = 'Public'; + UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template'; -- Update existing vm_template records with NULL type to "USER" UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL; + +-- remove unused config item +DELETE FROM `cloud`.`configuration` WHERE name = 'consoleproxy.cmd.port'; + +-- Drops the unused "backup_interval_type" column of the "cloud.backups" table +ALTER TABLE `cloud`.`backups` DROP COLUMN `backup_interval_type`; + +-- Update `user.password.reset.mail.template` configuration value to match new logic +UPDATE `cloud`.`configuration` +SET value = CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team') +WHERE name = 'user.password.reset.mail.template' + AND value IN (CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', 'http://{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team'), CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{domainUrl}}}{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team')); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 4cb9eb7cb2c4..478125335e70 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -117,3 +117,82 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tin --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); + +--- Quota tariff/usage mapping +CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_tariff_usage` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `tariff_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the tariff of the Quota usage detail calculated, foreign key to quota_tariff table', + `quota_usage_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the aggregation of Quota usage details, foreign key to quota_usage table', + `quota_used` decimal(20,8) NOT NULL COMMENT 'Amount of quota used', + PRIMARY KEY (`id`), + CONSTRAINT `fk_quota_tariff_usage__tariff_id` FOREIGN KEY (`tariff_id`) REFERENCES `cloud_usage`.`quota_tariff` (`id`), + CONSTRAINT `fk_quota_tariff_usage__quota_usage_id` FOREIGN KEY (`quota_usage_id`) REFERENCES `cloud_usage`.`quota_usage` (`id`)); + +-- Add the 'keep_mac_address_on_public_nic' column to the 'cloud.networks' and 'cloud.vpc' tables +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.networks', 'keep_mac_address_on_public_nic', 'TINYINT(1) NOT NULL DEFAULT 1'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc', 'keep_mac_address_on_public_nic', 'TINYINT(1) NOT NULL DEFAULT 1'); + +-- Creates the 'kvm.memory.dynamic.scaling.capacity' and, for already active ACS environments, +-- initializes it with the value of the setting 'vm.serviceoffering.ram.size.max' +INSERT INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `updated`, `scope`, `is_dynamic`, `group_id`, `subgroup_id`, `display_text`, `description`) +SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.memory.dynamic.scaling.capacity', `cfg`.`value`, 0, NULL, 4, 1, 6, 27, + 'KVM memory dynamic scaling capacity', 'Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. The ''kvm.memory.dynamic.scaling.capacity'' setting''s value will be used to define the value of the '''' element of domain XMLs. If it is set to a value less than or equal to ''0'', then the host''s memory capacity will be considered.' +FROM `cloud`.`configuration` `cfg` +WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.memory.dynamic.scaling.capacity') + AND `cfg`.`name` = 'vm.serviceoffering.ram.size.max'; + +-- Creates the 'kvm.cpu.dynamic.scaling.capacity' and, for already active ACS environments, +-- initializes it with the value of the setting 'vm.serviceoffering.cpu.cores.max' +INSERT INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `updated`, `scope`, `is_dynamic`, `group_id`, `subgroup_id`, `display_text`, `description`) +SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.cpu.dynamic.scaling.capacity', `cfg`.`value`, 0, NULL, 4, 1, 6, 27, + 'KVM CPU dynamic scaling capacity', 'Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. The ''kvm.cpu.dynamic.scaling.capacity'' setting''s value will be used to define the value of the '''' element of domain XMLs. If it is set to a value less than or equal to ''0'', then the host''s CPU cores capacity will be considered.' +FROM `cloud`.`configuration` `cfg` +WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.cpu.dynamic.scaling.capacity') + AND `cfg`.`name` = 'vm.serviceoffering.cpu.cores.max'; + +-- Add management_server_details table to allow ManagementServer scope configs +CREATE TABLE IF NOT EXISTS `management_server_details` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `management_server_id` bigint unsigned NOT NULL COMMENT 'management server the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user', + PRIMARY KEY (`id`), + CONSTRAINT `fk_management_server_details__management_server_id` FOREIGN KEY `fk_management_server_details__management_server_id`(`management_server_id`) REFERENCES `mshost`(`id`) ON DELETE CASCADE, + KEY `i_management_server_details__name__value` (`name`(128),`value`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Add checkpoint tracking fields to backups table for incremental backup support +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'from_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "Previous active checkpoint id for incremental backups"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'to_checkpoint_id', 'VARCHAR(255) DEFAULT NULL COMMENT "New checkpoint id created for the next incremental backup"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'checkpoint_create_time', 'BIGINT DEFAULT NULL COMMENT "Checkpoint creation timestamp from libvirt"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'host_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT "Host where backup is running"'); + +-- Create image_transfer table for per-disk image transfers +CREATE TABLE IF NOT EXISTS `cloud`.`image_transfer`( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) NOT NULL COMMENT 'uuid', + `account_id` bigint unsigned NOT NULL COMMENT 'Account ID', + `domain_id` bigint unsigned NOT NULL COMMENT 'Domain ID', + `data_center_id` bigint unsigned NOT NULL COMMENT 'Data Center ID', + `backup_id` bigint unsigned COMMENT 'Backup ID', + `volume_id` bigint unsigned NOT NULL COMMENT 'Volume ID', + `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', + `transfer_url` varchar(255) COMMENT 'ImageIO transfer URL', + `file` varchar(255) COMMENT 'File for the file backend', + `phase` varchar(20) NOT NULL COMMENT 'Transfer phase: initializing, transferring, finished, failed', + `socket` varchar(255) COMMENT 'Unix socket for nbd backend', + `direction` varchar(20) NOT NULL COMMENT 'Direction: upload, download', + `backend` varchar(20) NOT NULL COMMENT 'Backend: nbd, file', + `progress` int COMMENT 'Transfer progress percentage (0-100)', + `signed_ticket_id` varchar(255) COMMENT 'Signed ticket ID from ImageIO', + `created` datetime NOT NULL COMMENT 'date created', + `updated` datetime COMMENT 'date updated if not null', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + UNIQUE KEY `uuid` (`uuid`), + CONSTRAINT `fk_image_transfer__backup_id` FOREIGN KEY (`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__volume_id` FOREIGN KEY (`volume_id`) REFERENCES `volumes`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_image_transfer__host_id` FOREIGN KEY (`host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE, + INDEX `i_image_transfer__backup_id`(`backup_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 6f31fc17bce7..d77ba27689be 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -56,6 +56,7 @@ SELECT `vm_instance`.`display_vm` AS `display_vm`, `vm_instance`.`delete_protection` AS `delete_protection`, `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_display_name`, `vm_instance`.`pod_id` AS `pod_id`, `host_pod_ref`.`uuid` AS `pod_uuid`, `vm_instance`.`private_ip_address` AS `private_ip_address`, @@ -79,6 +80,7 @@ SELECT `vm_template`.`format` AS `template_format`, `vm_template`.`display_text` AS `template_display_text`, `vm_template`.`enable_password` AS `password_enabled`, + `vm_template`.`extension_id` AS `template_extension_id`, `iso`.`id` AS `iso_id`, `iso`.`uuid` AS `iso_uuid`, `iso`.`name` AS `iso_name`, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql new file mode 100644 index 000000000000..7ac001384e8e --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud_usage`.`quota_usage_view`; + +DROP VIEW IF EXISTS `cloud_usage`.`quota_usage_view`; +CREATE VIEW `cloud_usage`.`quota_usage_view` AS +SELECT qu.id, + qu.usage_item_id, + qu.zone_id, + qu.account_id, + qu.domain_id, + qu.usage_type, + qu.quota_used, + qu.start_date, + qu.end_date, + cu.usage_id AS resource_id, + cu.network_id as network_id, + cu.offering_id as offering_id +FROM `cloud_usage`.`quota_usage` qu +INNER JOIN `cloud_usage`.`cloud_usage` cu ON (cu.id = qu.usage_item_id); diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java index 9445efeb089c..6f153727ab76 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java @@ -41,6 +41,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; @@ -113,6 +114,69 @@ public void testListPoolIdsByVolumeCount_without_cluster_details() throws SQLExc verify(preparedStatementMock, times(1)).executeQuery(); } + @Test + public void findByInstanceAndNotState_queriesWithInstanceIdAndExcludedStates() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + + volumeDao.findByInstanceAndNotStates(42L, Volume.State.Ready); + + Mockito.verify(sc).setParameters("instanceId", 42L); + Mockito.verify(sc).setParameters("state", (Object[]) new Volume.State[]{Volume.State.Ready}); + } + + @Test + public void findByInstanceAndNotStates_withMultipleExcludedStates_passesAllStatesToCriteria() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + + volumeDao.findByInstanceAndNotStates(7L, Volume.State.Destroy, Volume.State.Expunged); + + Mockito.verify(sc).setParameters("instanceId", 7L); + Mockito.verify(sc).setParameters("state", + (Object[]) new Volume.State[]{Volume.State.Destroy, Volume.State.Expunged}); + } + + @Test + public void findByInstanceAndNotStates_returnsResultFromDao() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + VolumeVO vol = Mockito.mock(VolumeVO.class); + Mockito.doReturn(List.of(vol)).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + + List result = volumeDao.findByInstanceAndNotStates(1L, Volume.State.Ready); + + Assert.assertEquals(1, result.size()); + Assert.assertSame(vol, result.get(0)); + } + + @Test + public void findByInstanceAndNotStates_noMatchingVolumes_returnsEmptyList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).listBy(sc); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + + List result = volumeDao.findByInstanceAndNotStates(99L, Volume.State.Ready); + + Assert.assertTrue(result.isEmpty()); + } + @Test public void testSearchRemovedByVmsNoVms() { Assert.assertTrue(CollectionUtils.isEmpty(volumeDao.searchRemovedByVms( @@ -141,5 +205,4 @@ public void testSearchRemovedByVms() { Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), Mockito.eq(false)); } - } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 27995eb179af..ab64e4698f01 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -44,6 +44,7 @@ import com.cloud.upgrade.dao.Upgrade41120to41200; import com.cloud.upgrade.dao.Upgrade41510to41520; import com.cloud.upgrade.dao.Upgrade41610to41700; +import com.cloud.upgrade.dao.Upgrade42010to42100; import com.cloud.upgrade.dao.Upgrade452to453; import com.cloud.upgrade.dao.Upgrade453to460; import com.cloud.upgrade.dao.Upgrade460to461; @@ -380,4 +381,23 @@ public void isNotStandalone() throws SQLException { assertFalse("DatabaseUpgradeChecker should not be a standalone component", checker.isStandalone()); } + @Test + public void testCalculateUpgradePath42010to42100() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.20.1.0"); + assertNotNull(dbVersion); + + final CloudStackVersion currentVersion = CloudStackVersion.parse("4.21.0.0"); + assertNotNull(currentVersion); + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + + assertNotNull(upgrades); + assertEquals(1, upgrades.length); + assertTrue(upgrades[0] instanceof Upgrade42010to42100); + + assertArrayEquals(new String[]{"4.20.1.0", "4.21.0.0"}, upgrades[0].getUpgradableVersionRange()); + assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion()); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index 8028e78c9073..51db952eb613 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -570,18 +570,19 @@ public void updateRegisteredTemplateDetails_UpdatesTemplateSuccessfully() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + String templateName = "templateName"; + when(templateVO.getName()).thenReturn(templateName); GuestOSVO guestOS = Mockito.mock(GuestOSVO.class); when(templateDetails.getGuestOs()).thenReturn("Debian"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(templateDetails.getName()).thenReturn("templateName"); when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); when(guestOSDao.findOneByDisplayName("Debian")).thenReturn(guestOS); when(guestOS.getId()).thenReturn(10L); when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); @@ -590,7 +591,7 @@ public void updateRegisteredTemplateDetails_UpdatesTemplateSuccessfully() { verify(vmTemplateDao).update(templateVO.getId(), templateVO); verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); } @Test @@ -620,16 +621,17 @@ public void updateRegisteredTemplateDetails_SkipsGuestOSUpdateWhenNotFound() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + String templateName = "templateName"; + when(templateVO.getName()).thenReturn(templateName); when(templateDetails.getGuestOs()).thenReturn("NonExistentOS"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(templateDetails.getName()).thenReturn("templateName"); when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); when(guestOSDao.findOneByDisplayName("NonExistentOS")).thenReturn(null); when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); @@ -637,7 +639,7 @@ public void updateRegisteredTemplateDetails_SkipsGuestOSUpdateWhenNotFound() { verify(vmTemplateDao).update(templateVO.getId(), templateVO); verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); } @Test diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java index 7174336113b5..88f479c09045 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java @@ -120,7 +120,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { private final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error, Snapshot.State.Hidden); public SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { - List snaps = snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + List snaps = snapshotStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(snapshotId, DataStoreRole.Image, State.Ready, State.Hidden); for (SnapshotDataStoreVO ref : snaps) { if (zoneId == dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { return ref; diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java index e3f28a7012c2..31b13fc279e3 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java @@ -111,7 +111,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { FreezeThawVMAnswer freezeAnswer = null; FreezeThawVMCommand thawCmd = null; FreezeThawVMAnswer thawAnswer = null; - List forRollback = new ArrayList<>(); + List snapshotsForRollback = new ArrayList<>(); long startFreeze = 0; try { vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshotVO, VMSnapshot.Event.CreateRequested); @@ -165,7 +165,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { logger.info("The virtual machine is frozen"); for (VolumeInfo vol : vinfos) { long startSnapshtot = System.nanoTime(); - SnapshotInfo snapInfo = createDiskSnapshot(vmSnapshot, forRollback, vol); + SnapshotInfo snapInfo = createDiskSnapshot(vmSnapshot, snapshotsForRollback, vol); if (snapInfo == null) { thawAnswer = (FreezeThawVMAnswer) agentMgr.send(hostId, thawCmd); @@ -222,7 +222,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { } } if (!result) { - for (SnapshotInfo snapshotInfo : forRollback) { + for (SnapshotInfo snapshotInfo : snapshotsForRollback) { rollbackDiskSnapshot(snapshotInfo); } try { @@ -388,10 +388,16 @@ private long elapsedTime(long startTime) { //Rollback if one of disks snapshot fails protected void rollbackDiskSnapshot(SnapshotInfo snapshotInfo) { + if (snapshotInfo == null) { + return; + } Long snapshotID = snapshotInfo.getId(); SnapshotVO snapshot = snapshotDao.findById(snapshotID); + if (snapshot == null) { + return; + } deleteSnapshotByStrategy(snapshot); - logger.debug("Rollback is executed: deleting snapshot with id:" + snapshotID); + logger.debug("Rollback is executed: deleting snapshot with id: {}", snapshotID); } protected void deleteSnapshotByStrategy(SnapshotVO snapshot) { @@ -434,7 +440,7 @@ protected void revertDiskSnapshot(VMSnapshot vmSnapshot) { } } - protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List forRollback, VolumeInfo vol) { + protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List snapshotsForRollback, VolumeInfo vol) { String snapshotName = vmSnapshot.getId() + "_" + vol.getUuid(); SnapshotVO snapshot = new SnapshotVO(vol.getDataCenterId(), vol.getAccountId(), vol.getDomainId(), vol.getId(), vol.getDiskOfferingId(), snapshotName, (short) Snapshot.Type.GROUP.ordinal(), Snapshot.Type.GROUP.name(), vol.getSize(), vol.getMinIops(), vol.getMaxIops(), Hypervisor.HypervisorType.KVM, null); @@ -448,6 +454,7 @@ protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List forRollback = new ArrayList<>(); + List snapshotsForRollback = new ArrayList<>(); VolumeInfo vol = Mockito.mock(VolumeInfo.class); SnapshotInfo snapshotInfo = Mockito.mock(SnapshotInfo.class); SnapshotStrategy strategy = Mockito.mock(SnapshotStrategy.class); @@ -179,7 +179,7 @@ public void testCreateDiskSnapshotBasedOnStrategy() throws Exception { VMSnapshotDetailsVO vmDetails = new VMSnapshotDetailsVO(vmSnapshot.getId(), volUuid, String.valueOf(snapshot.getId()), false); when(vmSnapshotDetailsDao.persist(any())).thenReturn(vmDetails); - info = vmStrategy.createDiskSnapshot(vmSnapshot, forRollback, vol); + info = vmStrategy.createDiskSnapshot(vmSnapshot, snapshotsForRollback, vol); assertNotNull(info); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index cd9ab3adb210..4057f7a051bf 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -145,10 +145,10 @@ protected List reorderPoolsByCapacity(DeploymentPlan plan, List, Map> result = capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType); List poolIdsByCapacity = result.first(); @@ -185,7 +185,7 @@ protected List reorderPoolsByNumberOfVolumes(DeploymentPlan plan, L Long clusterId = plan.getClusterId(); List poolIdsByVolCount = volumeDao.listPoolIdsByVolumeCount(dcId, podId, clusterId, account.getAccountId()); - logger.debug(String.format("List of pools in ascending order of number of volumes for account [%s] is [%s].", account, poolIdsByVolCount)); + logger.debug("List of pools in ascending order of number of volumes for account [{}] is [{}].", account, poolIdsByVolCount); // now filter the given list of Pools by this ordered list Map poolMap = new HashMap<>(); @@ -206,16 +206,11 @@ protected List reorderPoolsByNumberOfVolumes(DeploymentPlan plan, L @Override public List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan, DiskProfile dskCh) { - if (logger.isTraceEnabled()) { - logger.trace("reordering pools"); - } if (pools == null) { - logger.trace("There are no pools to reorder; returning null."); + logger.info("There are no pools to reorder."); return null; } - if (logger.isTraceEnabled()) { - logger.trace(String.format("reordering %d pools", pools.size())); - } + logger.info("Reordering [{}] pools", pools.size()); Account account = null; if (vmProfile.getVirtualMachine() != null) { account = vmProfile.getOwner(); @@ -224,9 +219,7 @@ public List reorderPools(List pools, VirtualMachinePro pools = reorderStoragePoolsBasedOnAlgorithm(pools, plan, account); if (vmProfile.getVirtualMachine() == null) { - if (logger.isTraceEnabled()) { - logger.trace("The VM is null, skipping pools reordering by disk provisioning type."); - } + logger.info("The VM is null, skipping pool reordering by disk provisioning type."); return pools; } @@ -240,14 +233,10 @@ public List reorderPools(List pools, VirtualMachinePro List reorderStoragePoolsBasedOnAlgorithm(List pools, DeploymentPlan plan, Account account) { String volumeAllocationAlgorithm = VolumeOrchestrationService.VolumeAllocationAlgorithm.value(); - logger.debug("Using volume allocation algorithm {} to reorder pools.", volumeAllocationAlgorithm); + logger.info("Using volume allocation algorithm {} to reorder pools.", volumeAllocationAlgorithm); if (volumeAllocationAlgorithm.equals("random") || (account == null)) { reorderRandomPools(pools); } else if (StringUtils.equalsAny(volumeAllocationAlgorithm, "userdispersing", "firstfitleastconsumed")) { - if (logger.isTraceEnabled()) { - logger.trace("Using reordering algorithm {}", volumeAllocationAlgorithm); - } - if (volumeAllocationAlgorithm.equals("userdispersing")) { pools = reorderPoolsByNumberOfVolumes(plan, pools, account); } else { @@ -259,16 +248,15 @@ List reorderStoragePoolsBasedOnAlgorithm(List pools, D void reorderRandomPools(List pools) { StorageUtil.traceLogStoragePools(pools, logger, "pools to choose from: "); - if (logger.isTraceEnabled()) { - logger.trace("Shuffle this so that we don't check the pools in the same order. Algorithm == 'random' (or no account?)"); - } - StorageUtil.traceLogStoragePools(pools, logger, "pools to shuffle: "); + logger.trace("Shuffle this so that we don't check the pools in the same order. Algorithm == 'random' (or no account?)"); + logger.debug("Pools to shuffle: [{}]", pools); Collections.shuffle(pools, secureRandom); - StorageUtil.traceLogStoragePools(pools, logger, "shuffled list of pools to choose from: "); + logger.debug("Shuffled list of pools to choose from: [{}]", pools); } private List reorderPoolsByDiskProvisioningType(List pools, DiskProfile diskProfile) { if (diskProfile != null && diskProfile.getProvisioningType() != null && !diskProfile.getProvisioningType().equals(Storage.ProvisioningType.THIN)) { + logger.info("Reordering [{}] pools by disk provisioning type [{}].", pools.size(), diskProfile.getProvisioningType()); List reorderedPools = new ArrayList<>(); int preferredIndex = 0; for (StoragePool pool : pools) { @@ -282,22 +270,28 @@ private List reorderPoolsByDiskProvisioningType(List p reorderedPools.add(preferredIndex++, pool); } } + logger.debug("Reordered list of pools by disk provisioning type [{}]: [{}]", diskProfile.getProvisioningType(), reorderedPools); return reorderedPools; } else { + if (diskProfile == null) { + logger.info("Reordering pools by disk provisioning type wasn't necessary, since no disk profile was found."); + } else { + logger.debug("Reordering pools by disk provisioning type wasn't necessary, since the provisioning type is [{}].", diskProfile.getProvisioningType()); + } return pools; } } protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, DeploymentPlan plan) { - logger.debug(String.format("Checking if storage pool [%s] is suitable to disk [%s].", pool, dskCh)); + logger.debug("Checking if storage pool [{}] is suitable to disk [{}].", pool, dskCh); if (avoid.shouldAvoid(pool)) { - logger.debug(String.format("StoragePool [%s] is in avoid set, skipping this pool to allocation of disk [%s].", pool, dskCh)); + logger.debug("StoragePool [{}] is in avoid set, skipping this pool to allocation of disk [{}].", pool, dskCh); return false; } if (dskCh.requiresEncryption() && !pool.getPoolType().supportsEncryption()) { if (logger.isDebugEnabled()) { - logger.debug(String.format("Storage pool type '%s' doesn't support encryption required for volume, skipping this pool", pool.getPoolType())); + logger.debug("Storage pool type '[{}]' doesn't support encryption required for volume, skipping this pool", pool.getPoolType()); } return false; } @@ -319,8 +313,8 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, } if (!checkDiskProvisioningSupport(dskCh, pool)) { - logger.debug(String.format("Storage pool [%s] does not have support to disk provisioning of disk [%s].", pool, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(dskCh, - "type", "name", "diskOfferingId", "templateId", "volumeId", "provisioningType", "hyperType"))); + logger.debug("Storage pool [{}] does not have support to disk provisioning of disk [{}].", pool, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(dskCh, + "type", "name", "diskOfferingId", "templateId", "volumeId", "provisioningType", "hyperType")); return false; } @@ -332,7 +326,7 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, HostVO plannedHost = hostDao.findById(plan.getHostId()); if (!storageMgr.checkIfHostAndStoragePoolHasCommonStorageAccessGroups(plannedHost, pool)) { if (logger.isDebugEnabled()) { - logger.debug(String.format("StoragePool %s and host %s does not have matching storage access groups", pool, plannedHost)); + logger.debug("StoragePool [{}] and host [{}] does not have matching storage access groups", pool, plannedHost); } return false; } @@ -343,13 +337,13 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, if (!isTempVolume) { volume = volumeDao.findById(dskCh.getVolumeId()); if (!storageMgr.storagePoolCompatibleWithVolumePool(pool, volume)) { - logger.debug(String.format("Pool [%s] is not compatible with volume [%s], skipping it.", pool, volume)); + logger.debug("Pool [{}] is not compatible with volume [{}], skipping it.", pool, volume); return false; } } if (pool.isManaged() && !storageUtil.managedStoragePoolCanScale(pool, plan.getClusterId(), plan.getHostId())) { - logger.debug(String.format("Cannot allocate pool [%s] to volume [%s] because the max number of managed clustered filesystems has been exceeded.", pool, volume)); + logger.debug("Cannot allocate pool [{}] to volume [{}] because the max number of managed clustered filesystems has been exceeded.", pool, volume); return false; } @@ -358,13 +352,13 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, requestVolumeDiskProfilePairs.add(new Pair<>(volume, dskCh)); if (dskCh.getHypervisorType() == HypervisorType.VMware) { if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster && storageMgr.isStoragePoolDatastoreClusterParent(pool)) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is a parent datastore cluster.", pool, volume)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is a parent datastore cluster.", pool, volume); return false; } if (pool.getParent() != 0L) { StoragePoolVO datastoreCluster = storagePoolDao.findById(pool.getParent()); if (datastoreCluster == null || (datastoreCluster != null && datastoreCluster.getStatus() != StoragePoolStatus.Up)) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not in [%s] state.", datastoreCluster, volume, StoragePoolStatus.Up)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is not in [{}] state.", datastoreCluster, volume, StoragePoolStatus.Up); return false; } } @@ -374,11 +368,11 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, storageMgr.isStoragePoolCompliantWithStoragePolicy(dskCh.getDiskOfferingId(), pool) : storageMgr.isStoragePoolCompliantWithStoragePolicy(requestVolumeDiskProfilePairs, pool); if (!isStoragePoolStoragePolicyCompliance) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not compliant with the storage policy required by the volume.", pool, volume)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is not compliant with the storage policy required by the volume.", pool, volume); return false; } } catch (StorageUnavailableException e) { - logger.warn(String.format("Could not verify storage policy compliance against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + logger.warn("Could not verify storage policy compliance against storage pool [{}] due to exception [{}]", pool.getUuid(), e.getMessage()); return false; } } @@ -427,19 +421,19 @@ private boolean checkHypervisorCompatibility(HypervisorType hyperType, Volume.Ty protected void logDisabledStoragePools(long dcId, Long podId, Long clusterId, ScopeType scope) { List disabledPools = storagePoolDao.findDisabledPoolsByScope(dcId, podId, clusterId, scope); if (disabledPools != null && !disabledPools.isEmpty()) { - logger.trace(String.format("Ignoring pools [%s] as they are in disabled state.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(disabledPools))); + logger.trace("Ignoring pools [{}] as they are in disabled state.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(disabledPools)); } } protected void logStartOfSearch(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, int returnUpTo, boolean bypassStorageTypeCheck){ - logger.trace(String.format("%s is looking for storage pools that match the VM's disk profile [%s], virtual machine profile [%s] and " - + "deployment plan [%s]. Returning up to [%d] and bypassStorageTypeCheck [%s].", this.getClass().getSimpleName(), dskCh, vmProfile, plan, returnUpTo, bypassStorageTypeCheck)); + logger.trace("[{}] is looking for storage pools that match the VM's disk profile [{}], virtual machine profile [{}] and " + + "deployment plan [{}]. Returning up to [{}] and bypassStorageTypeCheck [{}].", this.getClass().getSimpleName(), dskCh, vmProfile, plan, returnUpTo, bypassStorageTypeCheck); } protected void logEndOfSearch(List storagePoolList) { - logger.debug(String.format("%s is returning [%s] suitable storage pools [%s].", this.getClass().getSimpleName(), storagePoolList.size(), - Arrays.toString(storagePoolList.toArray()))); + logger.debug("[{}] is returning [{}] suitable storage pools [{}].", this.getClass().getSimpleName(), storagePoolList.size(), + Arrays.toString(storagePoolList.toArray())); } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index a2e9eff2a08a..26b39e30776f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -230,8 +230,10 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); @@ -285,19 +286,22 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 436f991afbd1..8731e8791ddd 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -395,9 +395,9 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { } // Find out if the volume is at state of download_in_progress on secondary storage - VolumeDataStoreVO volumeStore = _volumeStoreDao.findByVolume(volume.getId()); - if (volumeStore != null) { - if (volumeStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { + VolumeDataStoreVO volumeOnImageStore = _volumeStoreDao.findByVolume(volume.getId()); + if (volumeOnImageStore != null) { + if (volumeOnImageStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { String msg = String.format("Volume: %s is currently being uploaded; can't delete it.", volume); logger.debug(msg); result.setSuccess(false); @@ -416,10 +416,10 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { if (!volumeExistsOnPrimary(vol)) { // not created on primary store - if (volumeStore == null) { + if (volumeOnImageStore == null) { // also not created on secondary store if (logger.isDebugEnabled()) { - logger.debug("Marking volume that was never created as destroyed: " + vol); + logger.debug("Marking volume that was never created as destroyed: {}", vol); } VMTemplateVO template = templateDao.findById(vol.getTemplateId()); if (template != null && !template.isDeployAsIs()) { @@ -435,11 +435,21 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { if (volume.getDataStore().getRole() == DataStoreRole.Image) { // no need to change state in volumes table volume.processEventOnly(Event.DestroyRequested); + if (volumeOnImageStore == null) { + logger.debug("Volume {} doesn't exist on image store, no need to delete", vol); + future.complete(result); + return future; + } } else if (volume.getDataStore().getRole() == DataStoreRole.Primary) { if (vol.getState() == Volume.State.Expunging) { logger.info("Volume {} is already in Expunging, retrying", volume); } volume.processEvent(Event.ExpungeRequested); + if (!volumeExistsOnPrimary(vol)) { + logger.debug("Volume {} doesn't exist on primary storage, no need to delete", vol); + future.complete(result); + return future; + } } DeleteVolumeContext context = new DeleteVolumeContext<>(null, vo, future); @@ -460,13 +470,11 @@ public void ensureVolumeIsExpungeReady(long volumeId) { private boolean volumeExistsOnPrimary(VolumeVO vol) { Long poolId = vol.getPoolId(); - if (poolId == null) { return false; } PrimaryDataStore primaryStore = dataStoreMgr.getPrimaryDataStore(poolId); - if (primaryStore == null) { return false; } @@ -476,8 +484,7 @@ private boolean volumeExistsOnPrimary(VolumeVO vol) { } String volumePath = vol.getPath(); - - if (volumePath == null || volumePath.trim().isEmpty()) { + if (StringUtils.isBlank(volumePath)) { return false; } diff --git a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java index 7c5692564c99..c9c48dbb179f 100644 --- a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java +++ b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java @@ -119,10 +119,10 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) { byte[] decodedUserData = null; // If GET, use 4K. If POST, support up to 1M. - if (httpmethod.equals(BaseCmd.HTTPMethod.GET)) { - decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); - } else if (httpmethod.equals(BaseCmd.HTTPMethod.POST)) { + if (BaseCmd.HTTPMethod.POST.equals(httpmethod)) { decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_POST_LENGTH, BaseCmd.HTTPMethod.POST); + } else { + decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); } // Re-encode so that the '=' paddings are added if necessary since 'isBase64' does not require it, but python does on the VR. diff --git a/framework/cluster/pom.xml b/framework/cluster/pom.xml index 2dd28e8e628f..75bcaf9a7ddf 100644 --- a/framework/cluster/pom.xml +++ b/framework/cluster/pom.xml @@ -48,6 +48,12 @@ cloud-api ${project.version} + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + compile + diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java new file mode 100644 index 000000000000..fcaa2a22e341 --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostDetailVO.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "management_server_details") +public class ManagementServerHostDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "management_server_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + @Column(name = "display") + private boolean display = true; + + public ManagementServerHostDetailVO(long poolId, String name, String value, boolean display) { + this.resourceId = poolId; + this.name = name; + this.value = value; + this.display = display; + } + + public ManagementServerHostDetailVO() { + } + + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return display; + } +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java new file mode 100644 index 000000000000..24fd60d21b3c --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.cluster.ManagementServerHostDetailVO; +import com.cloud.utils.db.GenericDao; + +public interface ManagementServerHostDetailsDao extends GenericDao, ResourceDetailsDao { +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java new file mode 100644 index 000000000000..5865bee0926b --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDetailsDaoImpl.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + +import com.cloud.cluster.ManagementServerHostDetailVO; + +public class ManagementServerHostDetailsDaoImpl extends ResourceDetailsDaoBase implements ManagementServerHostDetailsDao, ScopedConfigStorage { + + public ManagementServerHostDetailsDaoImpl() { + } + + @Override + public ConfigKey.Scope getScope() { + return ConfigKey.Scope.ManagementServer; + } + + @Override + public String getConfigValue(long id, String key) { + ManagementServerHostDetailVO vo = findDetail(id, key); + return vo == null ? null : vo.getValue(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new ManagementServerHostDetailVO(resourceId, key, value, display)); + } +} diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 15807eb26d42..3323d5c4d829 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -205,6 +205,12 @@ public void setJoinParameters(String joinName, String conditionName, Object... p } + public void setJoinParametersIfNotNull(String joinName, String conditionName, Object... params) { + if (ArrayUtils.isNotEmpty(params) && (params.length > 1 || params[0] != null)) { + setJoinParameters(joinName, conditionName, params); + } + } + public SearchCriteria getJoin(String joinName) { return _joins.get(joinName).getT(); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java index 5ab54149645e..9d76a7e6ec25 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java @@ -83,6 +83,12 @@ public class CreateExtensionCmd extends BaseCmd { description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].endpoint.url=urlvalue") protected Map details; + @Parameter(name = ApiConstants.RESERVED_RESOURCE_DETAILS, type = CommandType.STRING, + description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -115,6 +121,10 @@ public Map getDetails() { return convertDetailsToMap(details); } + public String getReservedResourceDetails() { + return reservedResourceDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java index ded07d2dd323..5baaea1709db 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java @@ -78,6 +78,12 @@ public class UpdateExtensionCmd extends BaseCmd { "if false or not set, no action)") private Boolean cleanupDetails; + @Parameter(name = ApiConstants.RESERVED_RESOURCE_DETAILS, type = CommandType.STRING, + description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -106,6 +112,10 @@ public Boolean isCleanupDetails() { return cleanupDetails; } + public String getReservedResourceDetails() { + return reservedResourceDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index 4171b9615fea..f6fd08b6da2c 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -216,6 +216,11 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject AccountService accountService; + // Map of in-built extension names and their reserved resource details that shouldn't be accessible to end-users + protected static final Map> INBUILT_RESERVED_RESOURCE_DETAILS = Map.of( + "proxmox", List.of("proxmox_vmid") + ); + private ScheduledExecutorService extensionPathStateCheckExecutor; protected String getDefaultExtensionRelativePath(String name) { @@ -563,6 +568,25 @@ protected void checkExtensionPathState(Extension extension, List reservedResourceDetails) { + ExtensionVO vo = extensionDao.findById(extensionId); + if (vo == null || vo.isUserDefined()) { + return; + } + String lowerName = StringUtils.defaultString(vo.getName()).toLowerCase(); + Optional>> match = INBUILT_RESERVED_RESOURCE_DETAILS.entrySet().stream() + .filter(e -> lowerName.contains(e.getKey().toLowerCase())) + .findFirst(); + if (match.isPresent()) { + Set existing = new HashSet<>(reservedResourceDetails); + for (String detailKey : match.get().getValue()) { + if (existing.add(detailKey)) { + reservedResourceDetails.add(detailKey); + } + } + } + } + @Override public String getExtensionsPath() { return externalProvisioner.getExtensionsPath(); @@ -577,6 +601,7 @@ public Extension createExtension(CreateExtensionCmd cmd) { String relativePath = cmd.getPath(); final Boolean orchestratorRequiresPrepareVm = cmd.isOrchestratorRequiresPrepareVm(); final String stateStr = cmd.getState(); + final String reservedResourceDetails = cmd.getReservedResourceDetails(); ExtensionVO extensionByName = extensionDao.findByName(name); if (extensionByName != null) { throw new CloudRuntimeException("Extension by name already exists"); @@ -624,6 +649,10 @@ public Extension createExtension(CreateExtensionCmd cmd) { ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, String.valueOf(orchestratorRequiresPrepareVm), false)); } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + detailsVOList.add(new ExtensionDetailsVO(extension.getId(), + ApiConstants.RESERVED_RESOURCE_DETAILS, reservedResourceDetails, false)); + } if (CollectionUtils.isNotEmpty(detailsVOList)) { extensionDetailsDao.saveDetails(detailsVOList); } @@ -704,6 +733,7 @@ public Extension updateExtension(UpdateExtensionCmd cmd) { final String stateStr = cmd.getState(); final Map details = cmd.getDetails(); final Boolean cleanupDetails = cmd.isCleanupDetails(); + final String reservedResourceDetails = cmd.getReservedResourceDetails(); final ExtensionVO extensionVO = extensionDao.findById(id); if (extensionVO == null) { throw new InvalidParameterValueException("Failed to find the extension"); @@ -732,7 +762,8 @@ public Extension updateExtension(UpdateExtensionCmd cmd) { throw new CloudRuntimeException(String.format("Failed to updated the extension: %s", extensionVO.getName())); } - updateExtensionsDetails(cleanupDetails, details, orchestratorRequiresPrepareVm, id); + updateExtensionsDetails(cleanupDetails, details, orchestratorRequiresPrepareVm, reservedResourceDetails, + id); return extensionVO; }); if (StringUtils.isNotBlank(stateStr)) { @@ -748,9 +779,11 @@ public Extension updateExtension(UpdateExtensionCmd cmd) { return result; } - protected void updateExtensionsDetails(Boolean cleanupDetails, Map details, Boolean orchestratorRequiresPrepareVm, long id) { + protected void updateExtensionsDetails(Boolean cleanupDetails, Map details, + Boolean orchestratorRequiresPrepareVm, String reservedResourceDetails, long id) { final boolean needToUpdateAllDetails = Boolean.TRUE.equals(cleanupDetails) || MapUtils.isNotEmpty(details); - if (!needToUpdateAllDetails && orchestratorRequiresPrepareVm == null) { + if (!needToUpdateAllDetails && orchestratorRequiresPrepareVm == null && + StringUtils.isBlank(reservedResourceDetails)) { return; } if (needToUpdateAllDetails) { @@ -761,6 +794,9 @@ protected void updateExtensionsDetails(Boolean cleanupDetails, Map detailsVOList.add( new ExtensionDetailsVO(id, key, value, false))); @@ -775,15 +811,29 @@ protected void updateExtensionsDetails(Boolean cleanupDetails, Map getExtensionReservedResourceDetails(long extensionId) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(extensionId, + ApiConstants.RESERVED_RESOURCE_DETAILS); + List reservedDetails = new ArrayList<>(); + if (detailsVO != null && StringUtils.isNotBlank(detailsVO.getValue())) { + String[] parts = detailsVO.getValue().split(","); + for (String part : parts) { + String trimmedPart = part.trim(); + if (StringUtils.isNotBlank(trimmedPart)) { + reservedDetails.add(trimmedPart); + } + } + } + addInbuiltExtensionReservedResourceDetails(extensionId, reservedDetails); + return reservedDetails; + } + @Override public boolean start() { long pathStateCheckInterval = PathStateCheckInterval.value(); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java index 2edb6ea48e3f..2f630966056c 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java @@ -94,4 +94,18 @@ public void testGetDetailsReturnsMap() { setField(cmd, "details", details); assertTrue(MapUtils.isNotEmpty(cmd.getDetails())); } + + @Test + public void getReservedResourceDetailsReturnsValueWhenSet() { + setField(cmd, "reservedResourceDetails", "detail1,detail2,detail3"); + String result = cmd.getReservedResourceDetails(); + assertEquals("detail1,detail2,detail3", result); + } + + @Test + public void getReservedResourceDetailsReturnsNullWhenNotSet() { + setField(cmd, "reservedResourceDetails", null); + String result = cmd.getReservedResourceDetails(); + assertNull(result); + } } diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java index f0a3a6fcf219..5c5c2014a529 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.springframework.test.util.ReflectionTestUtils.setField; import java.util.EnumSet; import java.util.HashMap; @@ -134,6 +135,20 @@ public void isCleanupDetailsReturnsValueWhenSet() { assertTrue(cmd.isCleanupDetails()); } + @Test + public void getReservedResourceDetailsReturnsValueWhenSet() { + setField(cmd, "reservedResourceDetails", "detail1,detail2,detail3"); + String result = cmd.getReservedResourceDetails(); + assertEquals("detail1,detail2,detail3", result); + } + + @Test + public void getReservedResourceDetailsReturnsNullWhenNotSet() { + setField(cmd, "reservedResourceDetails", null); + String result = cmd.getReservedResourceDetails(); + assertNull(result); + } + @Test public void executeSetsExtensionResponseWhenManagerSucceeds() { Extension extension = mock(Extension.class); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java index 085ae212b281..ff3fce06b006 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java @@ -23,11 +23,13 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -40,6 +42,7 @@ import java.io.File; import java.security.InvalidParameterException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -49,8 +52,6 @@ import java.util.Map; import java.util.UUID; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.user.AccountService; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; @@ -85,9 +86,11 @@ import org.apache.cloudstack.framework.extensions.dao.ExtensionResourceMapDetailsDao; import org.apache.cloudstack.framework.extensions.vo.ExtensionCustomActionDetailsVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionCustomActionVO; +import org.apache.cloudstack.framework.extensions.vo.ExtensionDetailsVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionResourceMapVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.collections.CollectionUtils; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -113,6 +116,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; @@ -122,6 +126,7 @@ import com.cloud.serializer.GsonHelper; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; +import com.cloud.user.AccountService; import com.cloud.utils.Pair; import com.cloud.utils.UuidUtils; import com.cloud.utils.db.EntityManager; @@ -664,6 +669,8 @@ public void testCreateExtension_Success() { when(cmd.getPath()).thenReturn(null); when(cmd.isOrchestratorRequiresPrepareVm()).thenReturn(null); when(cmd.getState()).thenReturn(null); + String reservedResourceDetails = "abc,xyz"; + when(cmd.getReservedResourceDetails()).thenReturn(reservedResourceDetails); when(extensionDao.findByName("ext1")).thenReturn(null); when(extensionDao.persist(any())).thenAnswer(inv -> { ExtensionVO extensionVO = inv.getArgument(0); @@ -671,11 +678,20 @@ public void testCreateExtension_Success() { return extensionVO; }); when(managementServerHostDao.listBy(any())).thenReturn(Collections.emptyList()); - + List detailsList = new ArrayList<>(); + doAnswer(inv -> { + List detailsVO = inv.getArgument(0); + detailsList.addAll(detailsVO); + return null; + }).when(extensionDetailsDao).saveDetails(anyList()); Extension ext = extensionsManager.createExtension(cmd); assertEquals("ext1", ext.getName()); verify(extensionDao).persist(any()); + assertTrue(CollectionUtils.isNotEmpty(detailsList)); + assertTrue(detailsList.stream() + .anyMatch(detail -> ApiConstants.RESERVED_RESOURCE_DETAILS.equals(detail.getName()) + && reservedResourceDetails.equals(detail.getValue()))); } @Test @@ -938,14 +954,32 @@ public void testUpdateExtension_InvalidState() { public void updateExtensionsDetails_SavesDetails_WhenDetailsProvided() { long extensionId = 10L; Map details = Map.of("foo", "bar", "baz", "qux"); - extensionsManager.updateExtensionsDetails(false, details, null, extensionId); + extensionsManager.updateExtensionsDetails(false, details, null, null, extensionId); verify(extensionDetailsDao).saveDetails(any()); } + @Test + public void updateExtensionsDetails_PersistReservedDetail_WhenProvided() { + long extensionId = 10L; + when(extensionDetailsDao.persist(any())).thenReturn(mock(ExtensionDetailsVO.class)); + extensionsManager.updateExtensionsDetails(false, null, null, "abc,xyz", extensionId); + verify(extensionDetailsDao).persist(any()); + } + + @Test + public void updateExtensionsDetails_UpdateReservedDetail_WhenProvided() { + long extensionId = 10L; + when(extensionDetailsDao.findDetail(anyLong(), eq(ApiConstants.RESERVED_RESOURCE_DETAILS))) + .thenReturn(mock(ExtensionDetailsVO.class)); + when(extensionDetailsDao.update(anyLong(), any())).thenReturn(true); + extensionsManager.updateExtensionsDetails(false, null, null, "abc,xyz", extensionId); + verify(extensionDetailsDao).update(anyLong(), any()); + } + @Test public void updateExtensionsDetails_DoesNothing_WhenDetailsAndCleanupAreNull() { long extensionId = 11L; - extensionsManager.updateExtensionsDetails(null, null, null, extensionId); + extensionsManager.updateExtensionsDetails(null, null, null, null, extensionId); verify(extensionDetailsDao, never()).removeDetails(anyLong()); verify(extensionDetailsDao, never()).saveDetails(any()); } @@ -953,7 +987,7 @@ public void updateExtensionsDetails_DoesNothing_WhenDetailsAndCleanupAreNull() { @Test public void updateExtensionsDetails_RemovesDetailsOnly_WhenCleanupIsTrue() { long extensionId = 12L; - extensionsManager.updateExtensionsDetails(true, null, null, extensionId); + extensionsManager.updateExtensionsDetails(true, null, null, null, extensionId); verify(extensionDetailsDao).removeDetails(extensionId); verify(extensionDetailsDao, never()).saveDetails(any()); } @@ -961,7 +995,7 @@ public void updateExtensionsDetails_RemovesDetailsOnly_WhenCleanupIsTrue() { @Test public void updateExtensionsDetails_PersistsOrchestratorFlag_WhenFlagIsNotNull() { long extensionId = 13L; - extensionsManager.updateExtensionsDetails(false, null, true, extensionId); + extensionsManager.updateExtensionsDetails(false, null, true, null, extensionId); verify(extensionDetailsDao).persist(any()); } @@ -970,7 +1004,7 @@ public void updateExtensionsDetails_ThrowsException_WhenPersistFails() { long extensionId = 14L; Map details = Map.of("foo", "bar"); doThrow(CloudRuntimeException.class).when(extensionDetailsDao).saveDetails(any()); - extensionsManager.updateExtensionsDetails(false, details, null, extensionId); + extensionsManager.updateExtensionsDetails(false, details, null, null, extensionId); } @Test @@ -1161,7 +1195,8 @@ public void testCreateExtensionResponse_HiddenDetailsOnly() { when(externalProvisioner.getExtensionPath("entry2.sh")).thenReturn("/some/path/entry2.sh"); Map hiddenDetails = Map.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, "false"); - when(extensionDetailsDao.listDetailsKeyPairs(2L, List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM))) + when(extensionDetailsDao.listDetailsKeyPairs(2L, List.of( + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, ApiConstants.RESERVED_RESOURCE_DETAILS))) .thenReturn(hiddenDetails); EnumSet viewDetails = EnumSet.noneOf(ApiConstants.ExtensionDetails.class); @@ -2069,4 +2104,118 @@ public void getCallerDetailsReturnsDetailsWithoutRoleWhenRoleIsNull() { } } + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenDetailsNotFound() { + long extensionId = 1L; + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(null); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenValueIsBlank() { + long extensionId = 2L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(" "); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsListOfTrimmedDetails() { + long extensionId = 3L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(" detail1 , detail2,detail3 "); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("detail1", result.get(0)); + assertEquals("detail2", result.get(1)); + assertEquals("detail3", result.get(2)); + } + + @Test + public void getExtensionReservedResourceDetailsHandlesEmptyPartsGracefully() { + long extensionId = 4L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn("detail1,,detail2, ,detail3"); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("detail1", result.get(0)); + assertEquals("detail2", result.get(1)); + assertEquals("detail3", result.get(2)); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenSplitResultsInNoParts() { + long extensionId = 5L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(","); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingWhenExtensionNotFound() { + when(extensionDao.findById(1L)).thenReturn(null); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(1L, reservedResourceDetails); + assertTrue(reservedResourceDetails.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingForUserDefinedExtension() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(true); + when(extensionDao.findById(2L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + reservedResourceDetails.add("existing-detail"); + extensionsManager.addInbuiltExtensionReservedResourceDetails(2L, reservedResourceDetails); + assertEquals(1, reservedResourceDetails.size()); + assertTrue(reservedResourceDetails.contains("existing-detail")); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingWhenNoMatchFound() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(false); + when(extension.getName()).thenReturn("no-such-inbuilt-key-expected"); + when(extensionDao.findById(3L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(3L, reservedResourceDetails); + assertTrue(reservedResourceDetails.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsAddedDetails() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(false); + Map.Entry> entry = + ExtensionsManagerImpl.INBUILT_RESERVED_RESOURCE_DETAILS.entrySet().iterator().next(); + when(extension.getName()).thenReturn(entry.getKey()); + when(extensionDao.findById(3L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(3L, reservedResourceDetails); + assertFalse(reservedResourceDetails.isEmpty()); + assertEquals(reservedResourceDetails.size(), entry.getValue().size()); + assertTrue(reservedResourceDetails.containsAll(entry.getValue())); + } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 9f7a4ad6e058..c334e91feb84 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -23,12 +23,30 @@ import com.cloud.utils.db.GenericDao; +import javax.annotation.Nullable; + public interface AsyncJobDao extends GenericDao { AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId); List findInstancePendingAsyncJobs(String instanceType, Long accountId); + /** + * Finds async job matching the given parameters. + * Non-null parameters are added to search criteria. + * Returns the most recent job by creation date. + *

+ * When searching by resourceId and resourceType, only one active job + * is expected per resource, so returning a single result is sufficient. + * + * @param id job ID + * @param resourceId resource ID (instanceId) + * @param resourceType resource type (instanceType) + * @return matching job or null + */ + @Nullable + AsyncJobVO findJob(Long id, Long resourceId, String resourceType); + AsyncJobVO findPseudoJob(long threadId, long msid); void cleanupPseduoJobs(long msid); @@ -50,4 +68,6 @@ public interface AsyncJobDao extends GenericDao { // Returns the number of pending jobs for the given Management server msids. // NOTE: This is the msid and NOT the id long countPendingNonPseudoJobs(Long... msIds); + + List listPendingJobIdsForAccount(long accountId); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index a2f1f36b8637..d8385e9aecd1 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -22,6 +22,8 @@ import java.util.List; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.jobs.JobInfo; @@ -45,6 +47,7 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements private final SearchBuilder expiringUnfinishedAsyncJobSearch; private final SearchBuilder expiringCompletedAsyncJobSearch; private final SearchBuilder failureMsidAsyncJobSearch; + private final SearchBuilder byIdResourceIdResourceTypeSearch; private final GenericSearchBuilder asyncJobTypeSearch; private final GenericSearchBuilder pendingNonPseudoAsyncJobsSearch; @@ -95,6 +98,12 @@ public AsyncJobDaoImpl() { failureMsidAsyncJobSearch.and("job_cmd", failureMsidAsyncJobSearch.entity().getCmd(), Op.IN); failureMsidAsyncJobSearch.done(); + byIdResourceIdResourceTypeSearch = createSearchBuilder(); + byIdResourceIdResourceTypeSearch.and("id", byIdResourceIdResourceTypeSearch.entity().getId(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.and("instanceId", byIdResourceIdResourceTypeSearch.entity().getInstanceId(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.and("instanceType", byIdResourceIdResourceTypeSearch.entity().getInstanceType(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.done(); + asyncJobTypeSearch = createSearchBuilder(Long.class); asyncJobTypeSearch.select(null, SearchCriteria.Func.COUNT, asyncJobTypeSearch.entity().getId()); asyncJobTypeSearch.and("job_info", asyncJobTypeSearch.entity().getCmdInfo(),Op.LIKE); @@ -140,6 +149,30 @@ public List findInstancePendingAsyncJobs(String instanceType, Long a return listBy(sc); } + @Override + public AsyncJobVO findJob(Long id, Long resourceId, String resourceType) { + SearchCriteria sc = byIdResourceIdResourceTypeSearch.create(); + + if (id == null && resourceId == null && StringUtils.isBlank(resourceType)) { + logger.debug("findJob called with all null parameters"); + return null; + } + + if (id != null) { + sc.setParameters("id", id); + } + if (resourceId != null && StringUtils.isNotBlank(resourceType)) { + sc.setParameters("instanceType", resourceType); + sc.setParameters("instanceId", resourceId); + } + Filter filter = new Filter(AsyncJobVO.class, "created", false, 0L, 1L); + List result = searchIncludingRemoved(sc, filter, Boolean.FALSE, false); + if (CollectionUtils.isNotEmpty(result)) { + return result.get(0); + } + return null; + } + @Override public AsyncJobVO findPseudoJob(long threadId, long msid) { SearchCriteria sc = pseudoJobSearch.create(); @@ -266,4 +299,14 @@ public long countPendingJobs(String havingInfo, String... cmds) { List results = customSearch(sc, null); return results.get(0); } + + @Override + public List listPendingJobIdsForAccount(long accountId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.selectFields(sb.entity().getId()); + SearchCriteria sc = sb.create(); + sc.setParameters("accountId", accountId); + return customSearch(sc, null); + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index 816144aa2f16..58f51eae9fcd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -150,8 +150,9 @@ protected void processQuotaBalanceForAccount(AccountVO accountVo, List> periods = accountQuotaUsages.stream() @@ -215,7 +216,7 @@ protected BigDecimal retrieveBalanceForUsageCalculation(long accountId, long dom logger.debug(String.format("Persisting the first quota balance [%s] for account [%s].", firstBalance, accountToString)); _quotaBalanceDao.saveQuotaBalance(firstBalance); } else { - QuotaBalanceVO lastRealBalance = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, startDate); + QuotaBalanceVO lastRealBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, startDate); if (lastRealBalance == null) { logger.warn("Account [{}] has quota usage entries, however it does not have a quota balance.", accountToString); @@ -244,7 +245,7 @@ protected void saveQuotaAccount(long accountId, BigDecimal aggregatedUsage, Date } protected BigDecimal aggregateCreditBetweenDates(Long accountId, Long domainId, Date startDate, Date endDate, String accountToString) { - List creditsReceived = _quotaBalanceDao.findCreditBalance(accountId, domainId, startDate, endDate); + List creditsReceived = _quotaBalanceDao.findCreditBalances(accountId, domainId, startDate, endDate); logger.debug("Account [{}] has [{}] credit entries before [{}].", accountToString, creditsReceived.size(), DateUtil.displayDateInTimezone(usageAggregationTimeZone, endDate)); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index a2fca7b80fd5..23020292027c 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -28,12 +28,14 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -66,6 +68,7 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostTagsDao; +import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.server.ResourceTag; @@ -191,6 +194,9 @@ public class PresetVariableHelper { @Inject ClusterDetailsDao clusterDetailsDao; + @Inject + VpcOfferingDao vpcOfferingDao; + protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value(); private List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); @@ -778,6 +784,30 @@ protected void loadPresetVariableValueForNetwork(UsageVO usageRecord, Value valu value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); + value.setResourceCounting(getPresetVariableValueNetworkResourceCounting(networkId)); + value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); + } + + protected ResourceCounting getPresetVariableValueNetworkResourceCounting(Long networkId) { + ResourceCounting resourceCounting = new ResourceCounting(); + List vmInstancesVO = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(networkId, VirtualMachine.Type.User); + int runningVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Running)).count(); + int stoppedVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Stopped)).count(); + + resourceCounting.setRunningVms(runningVms); + resourceCounting.setStoppedVms(stoppedVms); + return resourceCounting; + } + + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { + NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); + validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); + + GenericPresetVariable networkOffering = new GenericPresetVariable(); + networkOffering.setId(networkOfferingVo.getUuid()); + networkOffering.setName(networkOfferingVo.getName()); + + return networkOffering; } protected void loadPresetVariableValueForVpc(UsageVO usageRecord, Value value) { @@ -793,6 +823,18 @@ protected void loadPresetVariableValueForVpc(UsageVO usageRecord, Value value) { value.setId(vpc.getUuid()); value.setName(vpc.getName()); + value.setVpcOffering(getPresetVariableValueVpcOffering(vpc.getVpcOfferingId())); + } + + protected GenericPresetVariable getPresetVariableValueVpcOffering(Long vpcOfferingId) { + VpcOfferingVO vpcOfferingVo = vpcOfferingDao.findByIdIncludingRemoved(vpcOfferingId); + validateIfObjectIsNull(vpcOfferingVo, vpcOfferingId, "vpc offering"); + + GenericPresetVariable vpcOffering = new GenericPresetVariable(); + vpcOffering.setId(vpcOfferingVo.getUuid()); + vpcOffering.setName(vpcOfferingVo.getName()); + + return vpcOffering; } /** diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java new file mode 100644 index 000000000000..75049c3486a3 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + + +import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class ResourceCounting { + + @PresetVariableDefinition(description = "The number of running user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int runningVms; + @PresetVariableDefinition(description = "The number of stopped user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int stoppedVms; + + public int getRunningVms() { + return runningVms; + } + + public void setRunningVms(int runningVms) { + this.runningVms = runningVms; + } + + public int getStoppedVms() { + return stoppedVms; + } + + public void setStoppedVms(int stoppedVms) { + this.stoppedVms = stoppedVms; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index ac776d13c578..286fe5c60fd5 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -84,6 +84,9 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + @PresetVariableDefinition(description = "The amount of resources of the usage type.") + private ResourceCounting resourceCounting; + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; @@ -96,6 +99,12 @@ public class Value extends GenericPresetVariable { private String state; + @PresetVariableDefinition(description = "Network offering of the network.", supportedTypes = {QuotaTypes.NETWORK}) + private GenericPresetVariable networkOffering; + + @PresetVariableDefinition(description = "VPC offering of the VPC.", supportedTypes = {QuotaTypes.VPC}) + private GenericPresetVariable vpcOffering; + public Host getHost() { return host; } @@ -255,4 +264,28 @@ public String getState() { public void setState(String state) { this.state = state; } + + public ResourceCounting getResourceCounting() { + return resourceCounting; + } + + public void setResourceCounting(ResourceCounting resourceCounting) { + this.resourceCounting = resourceCounting; + } + + public GenericPresetVariable getNetworkOffering() { + return networkOffering; + } + + public void setNetworkOffering(GenericPresetVariable networkOffering) { + this.networkOffering = networkOffering; + } + + public GenericPresetVariable getVpcOffering() { + return vpcOffering; + } + + public void setVpcOffering(GenericPresetVariable vpcOffering) { + this.vpcOffering = vpcOffering; + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java index c694eaeefbe8..2964746bdf06 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java @@ -28,16 +28,14 @@ public interface QuotaBalanceDao extends GenericDao { QuotaBalanceVO saveQuotaBalance(QuotaBalanceVO qb); - List findCreditBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List findCreditBalances(Long accountId, Long domainId, Date startDate, Date endDate); - QuotaBalanceVO findLastBalanceEntry(Long accountId, Long domainId, Date beforeThis); + QuotaBalanceVO getLastQuotaBalanceEntry(Long accountId, Long domainId, Date beforeThis); QuotaBalanceVO findLaterBalanceEntry(Long accountId, Long domainId, Date afterThis); - List findQuotaBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List listQuotaBalances(Long accountId, Long domainId, Date startDate, Date endDate); - List lastQuotaBalanceVO(Long accountId, Long domainId, Date startDate); - - BigDecimal lastQuotaBalance(Long accountId, Long domainId, Date startDate); + BigDecimal getLastQuotaBalance(Long accountId, Long domainId); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java index 01272d1a6184..21553ebd27b0 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java @@ -18,11 +18,14 @@ import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -32,160 +35,104 @@ import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.db.TransactionStatus; @Component public class QuotaBalanceDaoImpl extends GenericDaoBase implements QuotaBalanceDao { + private static final Logger logger = LogManager.getLogger(QuotaBalanceDaoImpl.class); @Override - public QuotaBalanceVO findLastBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + public QuotaBalanceVO getLastQuotaBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + + if (beforeThis != null) { qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LT, beforeThis); - quotaBalanceEntries = search(qb.create(), filter); - return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; } + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO findLaterBalanceEntry(final Long accountId, final Long domainId, final Date afterThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); - quotaBalanceEntries = search(qb.create(), filter); - return quotaBalanceEntries.size() > 0 ? quotaBalanceEntries.get(0) : null; - } + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO saveQuotaBalance(final QuotaBalanceVO qb) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - return persist(qb); - } - }); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(qb)); } @Override - public List findCreditBalance(final Long accountId, final Long domainId, final Date lastbalancedate, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - if ((lastbalancedate != null) && (beforeThis != null) && lastbalancedate.before(beforeThis)) { - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, lastbalancedate, beforeThis); - return search(qb.create(), filter); - } else { - return new ArrayList(); - } + public List findCreditBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + if (ObjectUtils.anyNull(startDate, endDate) || startDate.after(endDate)) { + return new ArrayList<>(); } - }); - } - @Override - public List findQuotaBalance(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((startDate != null) && (endDate != null) && startDate.before(endDate)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } else { - return Collections. emptyList(); - } - quotaUsageRecords = listBy(qb.create()); - if (quotaUsageRecords.size() == 0) { - quotaUsageRecords.addAll(lastQuotaBalanceVO(accountId, domainId, startDate)); - } - return quotaUsageRecords; + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } + return search(qb.create(), filter); }); - } @Override - public List lastQuotaBalanceVO(final Long accountId, final Long domainId, final Date pivotDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - List trimmedRecords = new ArrayList(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 100L); - // ASSUMPTION there will be less than 100 continuous credit - // transactions - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((pivotDate != null)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, pivotDate); - } - quotaUsageRecords = search(qb.create(), filter); - - // get records before startDate to find start balance - for (QuotaBalanceVO entry : quotaUsageRecords) { - if (logger.isDebugEnabled()) { - logger.debug("FindQuotaBalance Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - trimmedRecords.add(entry); - } else { - trimmedRecords.add(entry); - break; // add only consecutive credit entries and last balance entry - } - } - return trimmedRecords; + public List listQuotaBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + if (startDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GTEQ, startDate); + } + if (endDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, endDate); } + + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true); + return listBy(qb.create(), filter); }); } @Override - public BigDecimal lastQuotaBalance(final Long accountId, final Long domainId, Date startDate) { - List quotaBalance = lastQuotaBalanceVO(accountId, domainId, startDate); - BigDecimal finalBalance = new BigDecimal(0); - if (quotaBalance.isEmpty()) { - logger.info("There are no balance entries on or before the requested date."); - return finalBalance; + public BigDecimal getLastQuotaBalance(final Long accountId, final Long domainId) { + QuotaBalanceVO quotaBalance = getLastQuotaBalanceEntry(accountId, domainId, null); + BigDecimal finalBalance = BigDecimal.ZERO; + Date startDate = DateUtils.addDays(new Date(), -1); + if (quotaBalance == null) { + logger.info("There are no balance entries for account [{}] and domain [{}]. Considering only new added credits.", accountId, domainId); + } else { + finalBalance = quotaBalance.getCreditBalance(); + startDate = quotaBalance.getUpdatedOn(); } - for (QuotaBalanceVO entry : quotaBalance) { - if (logger.isDebugEnabled()) { - logger.debug("lastQuotaBalance Entry=" + entry); - } - finalBalance = finalBalance.add(entry.getCreditBalance()); + + List credits = findCreditBalances(accountId, domainId, startDate, new Date()); + + for (QuotaBalanceVO credit : credits) { + finalBalance = finalBalance.add(credit.getCreditBalance()); } + return finalBalance; } + private QueryBuilder getQuotaBalanceQueryBuilder(Long accountId, Long domainId) { + QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); + qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); + qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); + return qb; + } + } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java new file mode 100644 index 000000000000..9684ca117b56 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java @@ -0,0 +1,29 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import java.util.List; + +public interface QuotaTariffUsageDao extends GenericDao { + + void persistQuotaTariffUsage(QuotaTariffUsageVO quotaTariffUsage); + + List listQuotaTariffUsages(Long quotaUsageId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java new file mode 100644 index 000000000000..556f552fed69 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Component +public class QuotaTariffUsageDaoImpl extends GenericDaoBase implements QuotaTariffUsageDao { + private SearchBuilder searchQuotaTariffUsages; + + @PostConstruct + public void init() { + searchQuotaTariffUsages = createSearchBuilder(); + searchQuotaTariffUsages.and("quotaUsageId", searchQuotaTariffUsages.entity().getQuotaUsageId(), SearchCriteria.Op.EQ); + searchQuotaTariffUsages.done(); + } + + @Override + public void persistQuotaTariffUsage(final QuotaTariffUsageVO quotaTariffUsage) { + logger.trace("Persisting quota tariff usage [{}].", quotaTariffUsage); + Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(quotaTariffUsage)); + } + + @Override + public List listQuotaTariffUsages(Long quotaUsageId) { + SearchCriteria sc = searchQuotaTariffUsages.create(); + sc.setParameters("quotaUsageId", quotaUsageId); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java new file mode 100644 index 000000000000..126fa11413f7 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; + +import java.util.Date; +import java.util.List; + +public interface QuotaUsageJoinDao extends GenericDao { + + List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java new file mode 100644 index 000000000000..b98ea2b3a5d2 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.Date; +import java.util.List; + +@Component +public class QuotaUsageJoinDaoImpl extends GenericDaoBase implements QuotaUsageJoinDao { + + private SearchBuilder searchQuotaUsages; + + private SearchBuilder searchQuotaUsagesJoinTariffUsages; + + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; + + @PostConstruct + public void init() { + searchQuotaUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsages); + searchQuotaUsages.done(); + + SearchBuilder searchQuotaTariffUsages = quotaTariffUsageDao.createSearchBuilder(); + searchQuotaTariffUsages.and("tariffId", searchQuotaTariffUsages.entity().getTariffId(), SearchCriteria.Op.EQ); + searchQuotaUsagesJoinTariffUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsagesJoinTariffUsages); + searchQuotaUsagesJoinTariffUsages.join("searchQuotaTariffUsages", searchQuotaTariffUsages, searchQuotaUsagesJoinTariffUsages.entity().getId(), + searchQuotaTariffUsages.entity().getQuotaUsageId(), JoinBuilder.JoinType.INNER); + searchQuotaUsagesJoinTariffUsages.done(); + } + + private void prepareQuotaUsageSearchBuilder(SearchBuilder searchBuilder) { + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("usageType", searchBuilder.entity().getUsageType(), SearchCriteria.Op.EQ); + searchBuilder.and("resourceId", searchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + searchBuilder.and("networkId", searchBuilder.entity().getNetworkId(), SearchCriteria.Op.EQ); + searchBuilder.and("offeringId", searchBuilder.entity().getOfferingId(), SearchCriteria.Op.EQ); + searchBuilder.and("startDate", searchBuilder.entity().getStartDate(), SearchCriteria.Op.BETWEEN); + searchBuilder.and("endDate", searchBuilder.entity().getEndDate(), SearchCriteria.Op.BETWEEN); + } + + @Override + public List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { + SearchCriteria sc = tariffId == null ? searchQuotaUsages.create() : searchQuotaUsagesJoinTariffUsages.create(); + + sc.setParametersIfNotNull("accountId", accountId); + sc.setParametersIfNotNull("domainId", domainId); + sc.setParametersIfNotNull("usageType", usageType); + sc.setParametersIfNotNull("resourceId", resourceId); + sc.setParametersIfNotNull("networkId", networkId); + sc.setParametersIfNotNull("offeringId", offeringId); + + if (ObjectUtils.allNotNull(startDate, endDate)) { + sc.setParameters("startDate", startDate, endDate); + sc.setParameters("endDate", startDate, endDate); + } + + sc.setJoinParametersIfNotNull("searchQuotaTariffUsages", "tariffId", tariffId); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java new file mode 100644 index 000000000000..4fa9e771713e --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java @@ -0,0 +1,86 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +@Entity +@Table(name = "quota_tariff_usage") +public class QuotaTariffUsageVO implements InternalIdentity { + @Id + @Column(name = "id") + private Long id; + + @Column(name = "tariff_id") + private Long tariffId; + + @Column(name = "quota_usage_id") + private Long quotaUsageId; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + public QuotaTariffUsageVO() { + quotaUsed = new BigDecimal(0); + } + + @Override + public long getId() { + return id; + } + + public Long getTariffId() { + return tariffId; + } + + public Long getQuotaUsageId() { + return quotaUsageId; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setId(Long id) { + this.id = id; + } + + public void setTariffId(Long tariffId) { + this.tariffId = tariffId; + } + + public void setQuotaUsageId(Long quotaUsageId) { + this.quotaUsageId = quotaUsageId; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + @Override + public String toString() { + return new ReflectionToStringBuilder(this, ToStringStyle.JSON_STYLE).toString(); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java new file mode 100644 index 000000000000..df9577e23c3e --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java @@ -0,0 +1,179 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.math.BigDecimal; +import java.util.Date; + +@Entity +@Table(name = "quota_usage_view") +public class QuotaUsageJoinVO implements InternalIdentity { + + @Id + @Column(name = "id", updatable = false, nullable = false) + private Long id; + + @Column(name = "zone_id") + private Long zoneId = null; + + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "domain_id") + private Long domainId = null; + + @Column(name = "usage_item_id") + private Long usageItemId; + + @Column(name = "usage_type") + private int usageType; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + @Column(name = "start_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startDate = null; + + @Column(name = "end_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endDate = null; + + @Column(name = "resource_id") + private Long resourceId = null; + + @Column(name = "network_id") + private Long networkId = null; + + @Column(name = "offering_id") + private Long offeringId = null; + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getZoneId() { + return zoneId; + } + + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getUsageItemId() { + return usageItemId; + } + + public void setUsageItemId(Long usageItemId) { + this.usageItemId = usageItemId; + } + + public int getUsageType() { + return usageType; + } + + public void setUsageType(int usageType) { + this.usageType = usageType; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + public Long getNetworkId() { + return networkId; + } + + public void setNetworkId(Long networkId) { + this.networkId = networkId; + } + + public Long getOfferingId() { + return offeringId; + } + + public void setOfferingId(Long offeringId) { + this.offeringId = offeringId; + } + + public QuotaUsageJoinVO () { + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "zoneId", "accountId", "domainId", "usageItemId", "usageType", "quotaUsed", "startDate", + "endDate", "resourceId"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java new file mode 100644 index 000000000000..924824231005 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java @@ -0,0 +1,62 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.vo; + +import java.util.Date; + +public class QuotaUsageResourceVO { + private String uuid; + private String name; + private Date removed; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public boolean isRemoved() { + return this.removed != null; + } + + public QuotaUsageResourceVO(String uuid, String name, Date removed) { + this.uuid = uuid; + this.name = name; + this.removed = removed; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 304b23b7220f..5ca2679c388c 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -26,7 +26,9 @@ - + + + diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index 85397503587e..bcdbc3b46cec 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,14 +31,22 @@ import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.VpcOfferingVO; +import com.cloud.network.vpc.VpcVO; +import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.cloudstack.quota.dao.NetworkDao; import org.apache.cloudstack.quota.dao.VmTemplateDao; +import org.apache.cloudstack.quota.dao.VpcDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -188,6 +196,15 @@ public class PresetVariableHelperTest { @Mock BackupOfferingDao backupOfferingDaoMock; + @Mock + NetworkDao networkDaoMock; + + @Mock + VpcDao vpcDaoMock; + + @Mock + VpcOfferingDao vpcOfferingDaoMock; + List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); List templateAndIsoUsageTypes = Arrays.asList(UsageTypes.TEMPLATE, UsageTypes.ISO); @@ -223,6 +240,10 @@ private Value getValueForTests() { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setState(Network.State.Implemented.toString()); + value.setResourceCounting(getResourceCountingForTests()); + value.setNetworkOffering(getNetworkOfferingForTests()); + value.setVpcOffering(getVpcOfferingForTests()); return value; } @@ -259,6 +280,13 @@ private Configuration getConfigurationForTests() { return configuration; } + private ResourceCounting getResourceCountingForTests() { + ResourceCounting resourceCounting = new ResourceCounting(); + resourceCounting.setRunningVms(1); + resourceCounting.setStoppedVms(1); + return resourceCounting; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -324,6 +352,20 @@ private DiskOfferingPresetVariables getDiskOfferingForTests() { return diskOffering; } + private GenericPresetVariable getNetworkOfferingForTests() { + GenericPresetVariable networkOffering = new GenericPresetVariable(); + networkOffering.setId("network_offering_id"); + networkOffering.setName("network_offering_name"); + return networkOffering; + } + + private GenericPresetVariable getVpcOfferingForTests() { + GenericPresetVariable vpcOffering = new GenericPresetVariable(); + vpcOffering.setId("vpc_offering_id"); + vpcOffering.setName("vpc_offering_name"); + return vpcOffering; + } + private void mockMethodValidateIfObjectIsNull() { Mockito.doNothing().when(presetVariableHelperSpy).validateIfObjectIsNull(Mockito.any(), Mockito.anyLong(), Mockito.anyString()); } @@ -1289,4 +1331,120 @@ public void testGetSnapshotImageStoreRefNotNull() { Mockito.when(imageStoreDaoMock.findById(1L)).thenReturn(store); Assert.assertNotNull(presetVariableHelperSpy.getSnapshotImageStoreRef(1L, 1L)); } + + @Test + public void loadPresetVariableValueForNetworkTestRecordIsNotANetworkDoNothing() { + getQuotaTypesForTests(UsageTypes.NETWORK).forEach(type -> { + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.loadPresetVariableValueForNetwork(usageVoMock, null); + }); + + Mockito.verifyNoInteractions(networkDaoMock); + } + + @Test + public void loadPresetVariableValueForNetworkTestRecordIsNetworkSetFields() { + Value expected = getValueForTests(); + + NetworkVO networkVoMock = Mockito.mock(NetworkVO.class); + Mockito.doReturn(networkVoMock).when(networkDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + mockMethodValidateIfObjectIsNull(); + + Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); + Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getResourceCounting()).when(presetVariableHelperSpy).getPresetVariableValueNetworkResourceCounting(Mockito.anyLong()); + Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); + Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); + + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForNetwork(usageVoMock, result); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getState(), result.getState()); + Assert.assertEquals(expected.getResourceCounting(), result.getResourceCounting()); + } + + @Test + public void getPresetVariableValueNetworkResourceCountingTestSetValueAndReturnObject() { + VMInstanceVO vmInstanceVoMock1 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock1.setState(VirtualMachine.State.Stopped); + + VMInstanceVO vmInstanceVoMock2 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock2.setState(VirtualMachine.State.Running); + + Mockito.doReturn(List.of(vmInstanceVoMock1, vmInstanceVoMock2)).when(vmInstanceDaoMock).listNonRemovedVmsByTypeAndNetwork(Mockito.anyLong(), Mockito.any()); + + mockMethodValidateIfObjectIsNull(); + + ResourceCounting expected = getResourceCountingForTests(); + + ResourceCounting result = presetVariableHelperSpy.getPresetVariableValueNetworkResourceCounting(1L); + + Assert.assertEquals(expected.getRunningVms(), result.getRunningVms()); + Assert.assertEquals(expected.getStoppedVms(), result.getStoppedVms()); + } + + @Test + public void loadPresetVariableValueForVpcTestRecordIsNotAVpcDoNothing() { + getQuotaTypesForTests(UsageTypes.VPC).forEach(type -> { + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); + }); + + Mockito.verifyNoInteractions(vpcDaoMock); + } + + @Test + public void loadPresetVariableValueForVpcTestRecordIsVpcSetFields() { + Value expected = getValueForTests(); + + VpcVO networkVoMock = Mockito.mock(VpcVO.class); + Mockito.doReturn(networkVoMock).when(vpcDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + mockMethodValidateIfObjectIsNull(); + + Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); + Mockito.doReturn(expected.getVpcOffering()).when(presetVariableHelperSpy).getPresetVariableValueVpcOffering(Mockito.anyLong()); + + Mockito.doReturn(UsageTypes.VPC).when(usageVoMock).getUsageType(); + + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, result); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getVpcOffering(), result.getVpcOffering()); + } + + @Test + public void getPresetVariableValueNetworkOfferingTestSetValuesAndReturnObject() { + NetworkOfferingVO networkOfferingVoMock = Mockito.mock(NetworkOfferingVO.class); + Mockito.doReturn(networkOfferingVoMock).when(networkOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + + GenericPresetVariable expected = getGenericPresetVariableForTests(); + Mockito.doReturn(expected.getId()).when(networkOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkOfferingVoMock).getName(); + + GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueNetworkOffering(1L); + + assertPresetVariableIdAndName(expected, result); + } + + @Test + public void getPresetVariableValueVpcOfferingTestSetValuesAndReturnObject() { + VpcOfferingVO vpcOfferingVoMock = Mockito.mock(VpcOfferingVO.class); + Mockito.doReturn(vpcOfferingVoMock).when(vpcOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + + GenericPresetVariable expected = getGenericPresetVariableForTests(); + Mockito.doReturn(expected.getId()).when(vpcOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(vpcOfferingVoMock).getName(); + + GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueVpcOffering(1L); + + assertPresetVariableIdAndName(expected, result); + } } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java new file mode 100644 index 000000000000..afdd88f8d1da --- /dev/null +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java @@ -0,0 +1,91 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; + +import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class QuotaBalanceDaoImplTest { + QuotaBalanceDaoImpl quotaBalanceDaoImplSpy = Mockito.spy(QuotaBalanceDaoImpl.class); + + @Mock + QuotaBalanceVO quotaBalanceVoMock; + + @Test + public void getLastQuotaBalanceTestLastEntryIsNullAndNoCreditsReturnsZero() { + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(1L, 2L); + + Assert.assertEquals(BigDecimal.ZERO, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryAndNoCredits() { + BigDecimal expected = BigDecimal.valueOf(-1542.46); + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryPlusCredits() { + BigDecimal balance = BigDecimal.valueOf(-1542.46); + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = balance.add(credit1).add(credit2); + + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(balance, credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryIsNullPlusCredits() { + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = credit1.add(credit2); + + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } +} diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 705959336f15..3ba2e4d5789e 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -83,6 +83,8 @@ Requires: (iptables-services or iptables) Requires: rng-tools Requires: (qemu-img or qemu-tools) Requires: python3-pip +Requires: python3-six +Requires: python3-protobuf Requires: python3-setuptools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Group: System Environment/Libraries @@ -115,7 +117,7 @@ Requires: ipset Requires: perl Requires: rsync Requires: cifs-utils -Requires: edk2-ovmf +Requires: (edk2-ovmf or qemu-ovmf-x86_64) Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) Requires: (qemu-img or qemu-tools) @@ -125,6 +127,8 @@ Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat +Requires: python3-libnbd +Requires: socat Provides: cloud-agent Group: System Environment/Libraries %description agent @@ -334,11 +338,11 @@ cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json ln -sf /etc/%{name}/ui/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json -# Package mysql-connector-python -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/e9/93/4860cebd5ad3ff2664ad3c966490ccb46e3b88458b2095145bca11727ca4/setuptools-47.3.1-py3-none-any.whl -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/32/27/1141a8232723dcb10a595cc0ce4321dcbbd5215300bf4acfc142343205bf/protobuf-3.19.6-py2.py3-none-any.whl +# Package mysql-connector-python (bundled to avoid dependency on external community repo) +# Version 8.0.31 is the last version supporting Python 3.6 (EL8) wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Version 8.3.0 supports Python 3.8 to 3.12 (EL9, EL10) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/53/ed/26a4b8cacb8852c6fd97d2d58a7f2591c41989807ea82bd8d9725a4e6937/mysql_connector_python-8.3.0-py2.py3-none-any.whl chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt @@ -455,8 +459,13 @@ then fi %post management -# Install mysql-connector-python -pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.19.6-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Install mysql-connector-python wheel +# Detect Python version to install compatible wheel +if python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 7) else 1)'; then + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.3.0-py2.py3-none-any.whl +else + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +fi /usr/bin/systemctl enable cloudstack-management > /dev/null 2>&1 || true /usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true diff --git a/packaging/suse15 b/packaging/suse15 deleted file mode 120000 index 4dad90d45e0c..000000000000 --- a/packaging/suse15 +++ /dev/null @@ -1 +0,0 @@ -el8 \ No newline at end of file diff --git a/packaging/suse15/cloud-ipallocator.rc b/packaging/suse15/cloud-ipallocator.rc new file mode 120000 index 000000000000..647598e6dc41 --- /dev/null +++ b/packaging/suse15/cloud-ipallocator.rc @@ -0,0 +1 @@ +../el8/cloud-ipallocator.rc \ No newline at end of file diff --git a/packaging/suse15/cloud.limits b/packaging/suse15/cloud.limits new file mode 120000 index 000000000000..37be77e3acf2 --- /dev/null +++ b/packaging/suse15/cloud.limits @@ -0,0 +1 @@ +../el8/cloud.limits \ No newline at end of file diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec new file mode 100644 index 000000000000..cdfc5a72a34e --- /dev/null +++ b/packaging/suse15/cloud.spec @@ -0,0 +1,750 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +%define __os_install_post %{nil} +%global debug_package %{nil} +%global __requires_exclude libc\\.so\\..*|libc\\.so\\.6\\(GLIBC_.*\\) +%define _binaries_in_noarch_packages_terminate_build 0 + +# DISABLE the post-percentinstall java repacking and line number stripping +# we need to find a way to just disable the java repacking and line number stripping, but not the autodeps + +Name: cloudstack +Summary: CloudStack IaaS Platform +#http://fedoraproject.org/wiki/PackageNamingGuidelines#Pre-Release_packages +%define _maventag %{_fullver} +Release: %{_rel} + +Version: %{_ver} +License: ASL 2.0 +Vendor: Apache CloudStack +Packager: Apache CloudStack +Group: System Environment/Libraries +# FIXME do groups for every single one of the subpackages +Source0: %{name}-%{_maventag}.tgz +BuildRoot: %{_tmppath}/%{name}-%{_maventag}-%{release}-build +BuildArch: noarch + +BuildRequires: (java-11-openjdk-devel or java-17-openjdk-devel or java-21-openjdk-devel) +#BuildRequires: ws-commons-util +BuildRequires: jpackage-utils +BuildRequires: gcc +BuildRequires: glibc-devel +BuildRequires: /usr/bin/mkisofs +BuildRequires: python3-setuptools +BuildRequires: wget +BuildRequires: nodejs + +%description +CloudStack is a highly-scalable elastic, open source, +intelligent IaaS cloud implementation. + +%package management +Summary: CloudStack management server UI +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Requires: python3 +Requires: bash +Requires: gawk +Requires: which +Requires: file +Requires: tar +Requires: bzip2 +Requires: gzip +Requires: unzip +Requires: (/sbin/mount.nfs or /usr/sbin/mount.nfs) +Requires: (openssh-clients or openssh) +Requires: (nfs-utils or nfs-client) +Requires: iproute +Requires: wget +Requires: (mysql or mariadb or mysql8.4) +Requires: sudo +Requires: /sbin/service +Requires: /sbin/chkconfig +Requires: /usr/bin/ssh-keygen +Requires: (genisoimage or mkisofs or xorrisofs) +Requires: ipmitool +Requires: %{name}-common = %{_ver} +Requires: (iptables-services or iptables) +Requires: rng-tools +Requires: (qemu-img or qemu-tools) +Requires: python3-pip +Requires: python3-six +Requires: python3-protobuf +Requires: python3-setuptools +Requires: (libgcrypt > 1.8.3 or libgcrypt20) +Group: System Environment/Libraries +%description management +The CloudStack management server is the central point of coordination, +management, and intelligence in CloudStack. + +%package common +Summary: Apache CloudStack common files and scripts +Requires: python3 +Group: System Environment/Libraries +%description common +The Apache CloudStack files shared between agent and management server +%global __requires_exclude libc\\.so\\..*|libc\\.so\\.6\\(GLIBC_.*\\)|^(libuuid\\.so\\.1|/usr/bin/python)$ + +%package agent +Summary: CloudStack Agent for KVM hypervisors +Requires: (openssh-clients or openssh) +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Requires: %{name}-common = %{_ver} +Requires: libvirt +Requires: libvirt-daemon-driver-storage-rbd +Requires: ebtables +Requires: iptables +Requires: ethtool +Requires: (net-tools or net-tools-deprecated) +Requires: iproute +Requires: ipset +Requires: perl +Requires: rsync +Requires: cifs-utils +Requires: (edk2-ovmf or qemu-ovmf-x86_64) +Requires: swtpm +Requires: (python3-libvirt or python3-libvirt-python) +Requires: (qemu-img or qemu-tools) +Requires: qemu-kvm +Requires: cryptsetup +Requires: rng-tools +Requires: (libgcrypt > 1.8.3 or libgcrypt20) +Requires: (selinux-tools if selinux-tools) +Requires: sysstat +Provides: cloud-agent +Group: System Environment/Libraries +%description agent +The CloudStack agent for KVM hypervisors + +%package baremetal-agent +Summary: CloudStack baremetal agent +Requires: tftp-server +Requires: xinetd +Requires: syslinux +Requires: chkconfig +Requires: dhcp +Requires: httpd +Group: System Environment/Libraries +%description baremetal-agent +The CloudStack baremetal agent + +%package usage +Summary: CloudStack Usage calculation server +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Group: System Environment/Libraries +%description usage +The CloudStack usage calculation service + +%package ui +Summary: CloudStack UI +Group: System Environment/Libraries +%description ui +The CloudStack UI + +%package marvin +Summary: Apache CloudStack Marvin library +Requires: python3-pip +Requires: gcc +Requires: python3-devel +Requires: libffi-devel +Requires: openssl-devel +Group: System Environment/Libraries +%description marvin +Apache CloudStack Marvin library + +%package integration-tests +Summary: Apache CloudStack Marvin integration tests +Requires: %{name}-marvin = %{_ver} +Group: System Environment/Libraries +%description integration-tests +Apache CloudStack Marvin integration tests + +%if "%{_ossnoss}" == "noredist" +%package mysql-ha +Summary: Apache CloudStack Balancing Strategy for MySQL +Group: System Environmnet/Libraries +%description mysql-ha +Apache CloudStack Balancing Strategy for MySQL + +%endif + +%prep +echo Doing CloudStack build + +%setup -q -n %{name}-%{_maventag} + +%build + +cp packaging/suse15/replace.properties build/replace.properties +echo VERSION=%{_maventag} >> build/replace.properties +echo PACKAGE=%{name} >> build/replace.properties +touch build/gitrev.txt +echo $(git rev-parse HEAD) > build/gitrev.txt + +if [ "%{_ossnoss}" == "NOREDIST" -o "%{_ossnoss}" == "noredist" ] ; then + echo "Adding noredist flag to the maven build" + FLAGS="$FLAGS -Dnoredist" +fi + +if [ "%{_sim}" == "SIMULATOR" -o "%{_sim}" == "simulator" ] ; then + echo "Adding simulator flag to the maven build" + FLAGS="$FLAGS -Dsimulator" +fi + +if [ \"%{_temp}\" != "" ]; then + echo "Adding flags to package requested templates" + FLAGS="$FLAGS `rpm --eval %{?_temp}`" +fi + +mvn -Psystemvm,developer $FLAGS clean package +cd ui && npm install && npm run build && cd .. + +%install +[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT} +# Common directories +mkdir -p ${RPM_BUILD_ROOT}%{_bindir} +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/ipallocator +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_initrddir} +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/default +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d + +# Common +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site +mkdir -p ${RPM_BUILD_ROOT}/usr/bin +cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +install -D systemvm/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/ +install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloud_utils.py +cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/ +python3 -m py_compile ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloud_utils.py +python3 -m compileall ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloudutils +cp build/gitrev.txt ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +cp packaging/suse15/cloudstack-sccs ${RPM_BUILD_ROOT}/usr/bin + +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco +cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco + +# Management +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/run +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel + +# Setup Jetty +ln -sf /etc/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/conf +ln -sf /var/log/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/logs + +install -D client/target/utilities/bin/cloud-migrate-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-migrate-databases +install -D client/target/utilities/bin/cloud-set-guest-password ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-password +install -D client/target/utilities/bin/cloud-set-guest-sshkey ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-sshkey +install -D client/target/utilities/bin/cloud-setup-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-databases +install -D client/target/utilities/bin/cloud-setup-encryption ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-encryption +install -D client/target/utilities/bin/cloud-setup-management ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-management +install -D client/target/utilities/bin/cloud-setup-baremetal ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-baremetal +install -D client/target/utilities/bin/cloud-sysvmadm ${RPM_BUILD_ROOT}%{_bindir}/%{name}-sysvmadm +install -D client/target/utilities/bin/cloud-update-xenserver-licenses ${RPM_BUILD_ROOT}%{_bindir}/%{name}-update-xenserver-licenses +# Bundle cmk in cloudstack-management +wget https://github.com/apache/cloudstack-cloudmonkey/releases/latest/download/cmk.linux.x86-64 -O ${RPM_BUILD_ROOT}%{_bindir}/cmk +chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk + +cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf +cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ +cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp +cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/ +cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/ +rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/config.json +ln -sf /etc/%{name}/management/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/config.json +mv ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/cloudstack-%{_maventag}.jar +cp client/target/lib/*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/ + +# Don't package the scripts in the management webapp +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/scripts +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/vms + +for name in db.properties server.properties log4j-cloud.xml environment.properties java.security.ciphers +do + cp client/target/conf/$name ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/$name +done + +ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j2.xml + +install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py +install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar + +install -D packaging/suse15/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator +install -D packaging/suse15/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud +install -D packaging/suse15/filelimit.conf ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d +install -D packaging/systemd/cloudstack-management.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-management.service +install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-management +install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management +touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid +#install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina +install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management + +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml + +# SystemVM template +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm +cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm/sha512sum.txt + +# Sample Extensions +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/extensions +cp -r extensions/* ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/extensions +ln -sf %{_sysconfdir}/%{name}/extensions ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/extensions + +# UI +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/ui +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ +cp -r client/target/classes/META-INF/webapp/WEB-INF ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui +cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/ui/ +cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ +rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json +ln -sf /etc/%{name}/ui/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json + +# Package mysql-connector-python (bundled to avoid dependency on external community repo) +# Version 8.0.31 is the last version supporting Python 3.6 (EL8) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Version 8.3.0 supports Python 3.8 to 3.12 (EL9, EL10) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/53/ed/26a4b8cacb8852c6fd97d2d58a7f2591c41989807ea82bd8d9725a4e6937/mysql_connector_python-8.3.0-py2.py3-none-any.whl + +chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent + +# KVM Agent +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/plugins +install -D packaging/systemd/cloudstack-agent.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-agent.service +install -D packaging/systemd/cloudstack-rolling-maintenance@.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-rolling-maintenance@.service +install -D packaging/systemd/cloudstack-agent.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-agent +install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/agent.properties +install -D agent/target/transformed/uefi.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/uefi.properties +install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties +install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml +install -D agent/target/transformed/cloud-setup-agent ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-agent +install -D agent/target/transformed/cloudstack-agent-upgrade ${RPM_BUILD_ROOT}%{_bindir}/%{name}-agent-upgrade +install -D agent/target/transformed/cloud-guest-tool ${RPM_BUILD_ROOT}%{_bindir}/%{name}-guest-tool +install -D agent/target/transformed/libvirtqemuhook ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook +install -D agent/target/transformed/rolling-maintenance ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/rolling-maintenance +install -D agent/target/transformed/cloud-ssh ${RPM_BUILD_ROOT}%{_bindir}/%{name}-ssh +install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d/%{name}-agent-profile.sh +install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent +install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar +cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +cp plugins/storage/volume/storpool/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +cp plugins/storage/volume/linstor/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib + +# Usage server +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib +install -D usage/target/cloud-usage-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/cloud-usage-%{_maventag}.jar +install -D usage/target/transformed/db.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/db.properties +install -D usage/target/transformed/log4j-cloud_usage.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/log4j-cloud.xml +cp usage/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ +cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ +install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service +install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/ +install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage + +# Marvin +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin +cp tools/marvin/dist/Marvin-*.tar.gz ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin/ + +# integration-tests +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests +cp -r test/integration/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests/ + +# MYSQL HA +if [ "x%{_ossnoss}" == "xnoredist" ] ; then + mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib + cp -r plugins/database/mysql-ha/target/cloud-plugin-database-mysqlha-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib +fi + +#License files from whisker +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-ui-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-ui-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE + +%clean +[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT} + +%posttrans common + +unalias cp +python_dir=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +if [ ! -z $python_dir ];then + cp -f -r /usr/share/cloudstack-common/python-site/* $python_dir/ +fi + +%preun management +/usr/bin/systemctl stop cloudstack-management || true +/usr/bin/systemctl disable cloudstack-management || true + +%pre management +id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -U -c "CloudStack unprivileged user" \ + -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud || true + +rm -rf %{_localstatedir}/cache/cloudstack + +# in case of upgrade to 4.9+ copy commands.properties if not exists in /etc/cloudstack/management/ +if [ "$1" == "2" ] ; then + if [ -f "%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties" ] && [ ! -f "%{_sysconfdir}/%{name}/management/commands.properties" ] ; then + cp -p %{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties %{_sysconfdir}/%{name}/management/commands.properties + fi +fi + +# Remove old tomcat symlinks and env config file +if [ -L "%{_datadir}/%{name}-management/lib" ] +then + rm -f %{_datadir}/%{name}-management/bin + rm -f %{_datadir}/%{name}-management/lib + rm -f %{_datadir}/%{name}-management/temp + rm -f %{_datadir}/%{name}-management/work + rm -f %{_sysconfdir}/default/%{name}-management +fi + +%post management +# Install mysql-connector-python wheel +# Detect Python version to install compatible wheel +if python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 7) else 1)'; then + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.3.0-py2.py3-none-any.whl +else + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +fi + +/usr/bin/systemctl enable cloudstack-management > /dev/null 2>&1 || true +/usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true + +grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "db.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "db.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" + +# Update DB properties having master and slave(s), with source and replica(s) respectively (for inclusiveness) +grep -s -q "^db.cloud.slaves=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.slaves=/db.cloud.replicas=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.cloud.secondsBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.secondsBeforeRetryMaster=/db.cloud.secondsBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.cloud.queriesBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.queriesBeforeRetryMaster=/db.cloud.queriesBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.slaves=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.slaves=/db.usage.replicas=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.secondsBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.secondsBeforeRetryMaster=/db.usage.secondsBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.queriesBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.queriesBeforeRetryMaster=/db.usage.queriesBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" + +if [ ! -f %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/vhd-util ] ; then + echo Please download vhd-util from http://download.cloudstack.org/tools/vhd-util and put it in + echo %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/ +fi + +if [ -f %{_sysconfdir}/sysconfig/%{name}-management ] ; then + rm -f %{_sysconfdir}/sysconfig/%{name}-management +fi + +chown -R cloud:cloud /var/log/cloudstack/management +chown -R cloud:cloud /usr/share/cloudstack-management/templates +find /usr/share/cloudstack-management/templates -type d -exec chmod 0770 {} \; + +systemctl daemon-reload + +%posttrans management +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text management +fi + +%preun agent +/sbin/service cloudstack-agent stop || true +if [ "$1" == "0" ] ; then + /sbin/chkconfig --del cloudstack-agent > /dev/null 2>&1 || true +fi + +%pre agent + +# save old configs if they exist (for upgrade). Otherwise we may lose them +# when the old packages are erased. There are a lot of properties files here. +if [ -d "%{_sysconfdir}/cloud" ] ; then + mv %{_sysconfdir}/cloud %{_sysconfdir}/cloud.rpmsave +fi + +%posttrans agent + +if [ "$1" == "2" ] ; then + echo "Running %{_bindir}/%{name}-agent-upgrade to update bridge name for upgrade from CloudStack 4.0.x (and before) to CloudStack 4.1 (and later)" + %{_bindir}/%{name}-agent-upgrade +fi +if [ ! -d %{_sysconfdir}/libvirt/hooks ] ; then + mkdir %{_sysconfdir}/libvirt/hooks +fi +cp -a ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook %{_sysconfdir}/libvirt/hooks/qemu +mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp +/usr/bin/systemctl restart libvirtd +/usr/bin/systemctl enable cloudstack-agent > /dev/null 2>&1 || true +/usr/bin/systemctl enable cloudstack-rolling-maintenance@p > /dev/null 2>&1 || true +/usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true + +# if saved agent.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/agent.properties %{_sysconfdir}/%{name}/agent/agent.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave +fi + +# if saved uefi.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/uefi.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/uefi.properties %{_sysconfdir}/%{name}/agent/uefi.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties.rpmsave +fi + +systemctl daemon-reload + +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text agent +fi + +%pre usage +id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -U -c "CloudStack unprivileged user" \ + -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true + +%preun usage +/sbin/service cloudstack-usage stop || true +if [ "$1" == "0" ] ; then + /sbin/chkconfig --del cloudstack-usage > /dev/null 2>&1 || true +fi + +%post usage +if [ -f "%{_sysconfdir}/%{name}/management/db.properties" ]; then + echo "Replacing usage server's db.properties with a link to the management server's db.properties" + rm -f %{_sysconfdir}/%{name}/usage/db.properties + ln -s %{_sysconfdir}/%{name}/management/db.properties %{_sysconfdir}/%{name}/usage/db.properties + /usr/bin/systemctl enable cloudstack-usage > /dev/null 2>&1 || true +fi + +if [ -f "%{_sysconfdir}/%{name}/management/key" ]; then + echo "Replacing usage server's key with a link to the management server's key" + rm -f %{_sysconfdir}/%{name}/usage/key + ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key +fi + +if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then + ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key +fi + +mkdir -p /usr/local/libexec +if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then + echo 1 > /usr/local/libexec/sanity-check-last-id +fi +chown cloud:cloud /usr/local/libexec/sanity-check-last-id + +%posttrans usage +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text usage +fi + +%post marvin +pip3 install --upgrade https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz + +#No default permission as the permission setup is complex +%files management +%defattr(-,root,root,-) +%dir %{_datadir}/%{name}-management +%dir %attr(0770,root,cloud) %{_localstatedir}/%{name}/mnt +%dir %attr(0770,cloud,cloud) %{_localstatedir}/%{name}/management +%dir %attr(0770,root,cloud) %{_localstatedir}/cache/%{name}/management +%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/management +%config(noreplace) %{_sysconfdir}/default/%{name}-management +%config(noreplace) %{_sysconfdir}/sudoers.d/%{name}-management +%config(noreplace) %{_sysconfdir}/security/limits.d/cloud +%config(noreplace) %{_sysconfdir}/systemd/system/%{name}-management.service.d +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/db.properties +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/server.properties +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/config.json +%config(noreplace) %{_sysconfdir}/%{name}/management/log4j-cloud.xml +%config(noreplace) %{_sysconfdir}/%{name}/management/log4j2.xml +%config(noreplace) %{_sysconfdir}/%{name}/management/environment.properties +%config(noreplace) %{_sysconfdir}/%{name}/management/java.security.ciphers +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management +%attr(0644,root,root) %{_unitdir}/%{name}-management.service +%attr(0755,cloud,cloud) %{_localstatedir}/run/%{name}-management.pid +%attr(0755,root,root) %{_bindir}/%{name}-setup-management +%attr(0755,root,root) %{_bindir}/%{name}-update-xenserver-licenses +%{_datadir}/%{name}-management/conf +%{_datadir}/%{name}-management/lib/*.jar +%{_datadir}/%{name}-management/logs +%{_datadir}/%{name}-management/templates +%{_datadir}/%{name}-management/extensions +%attr(0755,root,root) %{_bindir}/%{name}-setup-databases +%attr(0755,root,root) %{_bindir}/%{name}-migrate-databases +%attr(0755,root,root) %{_bindir}/%{name}-set-guest-password +%attr(0755,root,root) %{_bindir}/%{name}-set-guest-sshkey +%attr(0755,root,root) %{_bindir}/%{name}-sysvmadm +%attr(0755,root,root) %{_bindir}/%{name}-setup-encryption +%attr(0755,root,root) %{_bindir}/cmk +%{_datadir}/%{name}-management/cks/conf/*.yml +%{_datadir}/%{name}-management/setup/*.sql +%{_datadir}/%{name}-management/setup/*.sh +%{_datadir}/%{name}-management/setup/server-setup.xml +%{_datadir}/%{name}-management/webapp/* +%dir %attr(0770, cloud, cloud) %{_datadir}/%{name}-management/templates +%dir %attr(0770, cloud, cloud) %{_datadir}/%{name}-management/templates/systemvm +%attr(0644, cloud, cloud) %{_datadir}/%{name}-management/templates/systemvm/* +%attr(0755,root,root) %{_bindir}/%{name}-external-ipallocator.py +%attr(0755,root,root) %{_initrddir}/%{name}-ipallocator +%dir %attr(0770,root,root) %{_localstatedir}/log/%{name}/ipallocator +%{_defaultdocdir}/%{name}-management-%{version}/LICENSE +%{_defaultdocdir}/%{name}-management-%{version}/NOTICE +%{_datadir}/%{name}-management/setup/wheel/*.whl +%dir %attr(0755,cloud,cloud) %{_sysconfdir}/%{name}/extensions +%attr(0755,cloud,cloud) %{_sysconfdir}/%{name}/extensions/* + +%files agent +%attr(0755,root,root) %{_bindir}/%{name}-setup-agent +%attr(0755,root,root) %{_bindir}/%{name}-agent-upgrade +%attr(0755,root,root) %{_bindir}/%{name}-guest-tool +%attr(0755,root,root) %{_bindir}/%{name}-ssh +%attr(0644,root,root) %{_unitdir}/%{name}-agent.service +%attr(0644,root,root) %{_unitdir}/%{name}-rolling-maintenance@.service +%config(noreplace) %{_sysconfdir}/default/%{name}-agent +%attr(0644,root,root) %{_sysconfdir}/profile.d/%{name}-agent-profile.sh +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-agent +%attr(0755,root,root) %{_datadir}/%{name}-common/scripts/network/cisco +%config(noreplace) %{_sysconfdir}/%{name}/agent +%dir %{_localstatedir}/log/%{name}/agent +%attr(0644,root,root) %{_datadir}/%{name}-agent/lib/*.jar +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/libvirtqemuhook +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/rolling-maintenance +%dir %{_datadir}/%{name}-agent/plugins +%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE +%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE + +%files common +%dir %attr(0755,root,root) %{_datadir}/%{name}-common/python-site/cloudutils +%dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms +%attr(0755,root,root) %{_datadir}/%{name}-common/scripts +%attr(0755,root,root) /usr/bin/cloudstack-sccs +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/agent.zip +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/cloud-scripts.tgz +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/patch-sysvms.sh +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloud_utils.py +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/__pycache__/* +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloudutils/* +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/%{name}-utils.jar +%{_defaultdocdir}/%{name}-common-%{version}/LICENSE +%{_defaultdocdir}/%{name}-common-%{version}/NOTICE + +%files ui +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/ui/config.json +%{_datadir}/%{name}-ui/* +%{_defaultdocdir}/%{name}-ui-%{version}/LICENSE +%{_defaultdocdir}/%{name}-ui-%{version}/NOTICE + +%files usage +%attr(0644,root,root) %{_unitdir}/%{name}-usage.service +%config(noreplace) %{_sysconfdir}/default/%{name}-usage +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage +%attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar +%attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar +%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage +%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/db.properties +%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/log4j-cloud.xml +%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE +%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE + +%files marvin +%attr(0644,root,root) %{_datadir}/%{name}-marvin/Marvin*.tar.gz +%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE +%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE + +%files integration-tests +%attr(0755,root,root) %{_datadir}/%{name}-integration-tests/* +%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE +%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE + +%if "%{_ossnoss}" == "noredist" +%files mysql-ha +%defattr(0644,cloud,cloud,0755) +%attr(0644,root,root) %{_datadir}/%{name}-mysql-ha/lib/* +%endif + +%files baremetal-agent +%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal + +%changelog +* Thu Dec 22 2022 Rohit Yadav 4.18.0 +- Add support for EL9 + +* Fri Oct 14 2022 Daan Hoogland 4.18.0 +- initialising sanity check pointer file + +* Tue Jun 29 2021 David Jumani 4.16.0 +- Adding SUSE 15 support + +* Thu Apr 30 2015 Rohit Yadav 4.6.0 +- Remove awsapi package + +* Wed Nov 19 2014 Hugo Trippaers 4.6.0 +- Create a specific spec for CentOS 7 + +* Fri Jul 4 2014 Hugo Trippaers 4.5.0 +- Add a package for the mysql ha module + +* Fri Oct 5 2012 Hugo Trippaers 4.1.0 +- new style spec file diff --git a/packaging/suse15/cloudstack-agent.te b/packaging/suse15/cloudstack-agent.te new file mode 120000 index 000000000000..30e123f6cba5 --- /dev/null +++ b/packaging/suse15/cloudstack-agent.te @@ -0,0 +1 @@ +../el8/cloudstack-agent.te \ No newline at end of file diff --git a/packaging/suse15/cloudstack-sccs b/packaging/suse15/cloudstack-sccs new file mode 120000 index 000000000000..b9e6ed9dc085 --- /dev/null +++ b/packaging/suse15/cloudstack-sccs @@ -0,0 +1 @@ +../el8/cloudstack-sccs \ No newline at end of file diff --git a/packaging/suse15/filelimit.conf b/packaging/suse15/filelimit.conf new file mode 120000 index 000000000000..c71688ea640a --- /dev/null +++ b/packaging/suse15/filelimit.conf @@ -0,0 +1 @@ +../el8/filelimit.conf \ No newline at end of file diff --git a/packaging/suse15/replace.properties b/packaging/suse15/replace.properties new file mode 100644 index 000000000000..b1900af83406 --- /dev/null +++ b/packaging/suse15/replace.properties @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DBUSER=cloud +DBPW=cloud +DBROOTPW= +MSLOG=vmops.log +APISERVERLOG=api.log +DBHOST=localhost +DBDRIVER=jdbc:mysql +COMPONENTS-SPEC=components-premium.xml +REMOTEHOST=localhost +AGENTCLASSPATH= +AGENTLOG=/var/log/cloudstack/agent/agent.log +AGENTLOGDIR=/var/log/cloudstack/agent/ +AGENTSYSCONFDIR=/etc/cloudstack/agent +APISERVERLOG=/var/log/cloudstack/management/apilog.log +BINDIR=/usr/bin +COMMONLIBDIR=/usr/share/cloudstack-common +CONFIGUREVARS= +DEPSCLASSPATH= +DOCDIR= +IPALOCATORLOG=/var/log/cloudstack/management/ipallocator.log +JAVADIR=/usr/share/java +LIBEXECDIR=/usr/libexec +LOCKDIR=/var/lock +MSCLASSPATH= +MSCONF=/etc/cloudstack/management +MSENVIRON=/usr/share/cloudstack-management +MSLOG=/var/log/cloudstack/management/management-server.log +MSLOGDIR=/var/log/cloudstack/management/ +MSMNTDIR=/var/cloudstack/mnt +MSUSER=cloud +PIDDIR=/var/run +PLUGINJAVADIR=/usr/share/cloudstack-management/plugin +PREMIUMJAVADIR=/usr/share/cloudstack-management/premium +PYTHONDIR=/usr/share/cloudstack-common/python-site/ +SERVERSYSCONFDIR=/etc/sysconfig +SETUPDATADIR=/usr/share/cloudstack-management/setup +SYSCONFDIR=/etc/sysconfig +SYSTEMCLASSPATH= +SYSTEMJARS= +USAGECLASSPATH= +USAGELOG=/var/log/cloudstack/usage/usage.log +USAGESYSCONFDIR=/etc/sysconfig +EXTENSIONSDEPLOYMENTMODE=production +GUESTNVRAMTEMPLATELEGACY=/usr/share/qemu/ovmf-x86_64-vars.bin +GUESTLOADERLEGACY=/usr/share/qemu/ovmf-x86_64-code.bin +GUESTNVRAMTEMPLATESECURE=/usr/share/qemu/ovmf-x86_64-ms-vars.bin +GUESTLOADERSECURE=/usr/share/qemu/ovmf-x86_64-ms-code.bin +GUESTNVRAMPATH=/var/lib/libvirt/qemu/nvram/ diff --git a/packaging/systemd/cloudstack-management.default b/packaging/systemd/cloudstack-management.default index 994a1ee86997..a41338beda68 100644 --- a/packaging/systemd/cloudstack-management.default +++ b/packaging/systemd/cloudstack-management.default @@ -17,7 +17,7 @@ JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED" -CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-mysql-ha/lib/*" +CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/cloudstack-mysql-ha/lib/*" BOOTSTRAP_CLASS=org.apache.cloudstack.ServerDaemon diff --git a/packaging/systemd/cloudstack-usage.default b/packaging/systemd/cloudstack-usage.default index 493f40c277a2..36b71ac3e0d6 100644 --- a/packaging/systemd/cloudstack-usage.default +++ b/packaging/systemd/cloudstack-usage.default @@ -17,7 +17,7 @@ JAVA_OPTS="-Xms256m -Xmx2048m --add-opens=java.base/java.lang=ALL-UNNAMED" -CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar" +CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage" JAVA_CLASS=com.cloud.usage.UsageServer diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index df9336026f4d..a7121f149e30 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -226,12 +226,12 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce } catch (AgentUnavailableException e) { logger.error("Unable to contact backend control plane to initiate backup for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { logger.error("Operation to initiate backup timed out for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } @@ -249,12 +249,12 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce } else { logger.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); if (answer.getNeedsCleanup()) { - logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state. Backup should be manually deleted to free up the space", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Error); backupDao.update(backupVO.getId(), backupVO); } else { backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); } return new Pair<>(false, null); } diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java index 5ff036fef12f..d018d488c64a 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java @@ -40,17 +40,17 @@ public final class RootCACustomTrustManager implements X509TrustManager { private boolean authStrictness = true; private boolean allowExpiredCertificate = true; private CrlDao crlDao; - private X509Certificate caCertificate; + private List caCertificates; private Map activeCertMap; - public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map activeCertMap, final X509Certificate caCertificate, final CrlDao crlDao) { + public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map activeCertMap, final List caCertificates, final CrlDao crlDao) { if (StringUtils.isNotEmpty(clientAddress)) { this.clientAddress = clientAddress.replace("/", "").split(":")[0]; } this.authStrictness = authStrictness; this.allowExpiredCertificate = allowExpiredCertificate; this.activeCertMap = activeCertMap; - this.caCertificate = caCertificate; + this.caCertificates = caCertificates; this.crlDao = crlDao; } @@ -151,6 +151,6 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr @Override public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[]{caCertificate}; + return caCertificates.toArray(new X509Certificate[0]); } } diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java index 25c45ed2a102..afb4f561160e 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -40,7 +40,6 @@ import java.security.spec.InvalidKeySpecException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Enumeration; import java.util.HashSet; import java.util.List; @@ -60,6 +59,7 @@ import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.ValidatedConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.KeyStoreUtils; @@ -92,6 +92,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con private static KeyPair caKeyPair = null; private static X509Certificate caCertificate = null; + private static List caCertificates = null; private static KeyStore managementKeyStore = null; @Inject @@ -103,20 +104,25 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con /////////////// Root CA Settings /////////////////// //////////////////////////////////////////////////// - private static ConfigKey rootCAPrivateKey = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.private.key", - null, - "The ROOT CA private key.", true); - - private static ConfigKey rootCAPublicKey = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.public.key", - null, - "The ROOT CA public key.", true); - - private static ConfigKey rootCACertificate = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.ca.certificate", - null, - "The ROOT CA certificate.", true); + private static ConfigKey rootCAPrivateKey = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.private.key", null, + "The ROOT CA private key in PEM format. " + + "When set along with the public key and certificate, CloudStack uses this custom CA instead of auto-generating one. " + + "All three ca.plugin.root.* keys must be set together. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validatePrivateKeyPem); + + private static ConfigKey rootCAPublicKey = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.public.key", null, + "The ROOT CA public key in PEM format (X.509/SPKI: must start with '-----BEGIN PUBLIC KEY-----'). " + + "Required when providing a custom CA. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validatePublicKeyPem); + + private static ConfigKey rootCACertificate = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.ca.certificate", null, + "The CA certificate(s) in PEM format (must start with '-----BEGIN CERTIFICATE-----'). " + + "For intermediate CAs, concatenate the signing cert first, followed by intermediate(s) and root. " + + "Required when providing a custom CA. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validateCACertificatePem); private static ConfigKey rootCAIssuerDN = new ConfigKey<>("Advanced", String.class, "ca.plugin.root.issuer.dn", @@ -151,7 +157,7 @@ private Certificate generateCertificate(final List domainNames, final Li caCertificate, caKeyPair, keyPair.getPublic(), subject, CAManager.CertSignatureAlgorithm.value(), validityDays, domainNames, ipAddresses); - return new Certificate(clientCertificate, keyPair.getPrivate(), Collections.singletonList(caCertificate)); + return new Certificate(clientCertificate, keyPair.getPrivate(), caCertificates); } private Certificate generateCertificateUsingCsr(final String csr, final List names, final List ips, final int validityDays) throws NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, CertificateException, SignatureException, IOException, OperatorCreationException { @@ -205,7 +211,7 @@ private Certificate generateCertificateUsingCsr(final String csr, final List getCaCertificate() { - return Collections.singletonList(caCertificate); + return caCertificates; } @Override @@ -254,8 +260,8 @@ public boolean revokeCertificate(final BigInteger certSerial, final String certC private KeyStore getCaKeyStore() throws CertificateException, NoSuchAlgorithmException, IOException, KeyStoreException { final KeyStore ks = KeyStore.getInstance("JKS"); ks.load(null, null); - if (caKeyPair != null && caCertificate != null) { - ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), new X509Certificate[]{caCertificate}); + if (caKeyPair != null && CollectionUtils.isNotEmpty(caCertificates)) { + ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), caCertificates.toArray(new X509Certificate[0])); } else { return null; } @@ -274,7 +280,7 @@ public SSLEngine createSSLEngine(final SSLContext sslContext, final String remot final boolean authStrictness = rootCAAuthStrictness.value(); final boolean allowExpiredCertificate = rootCAAllowExpiredCert.value(); - TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificate, crlDao)}; + TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificates, crlDao)}; sslContext.init(kmf.getKeyManagers(), tms, new SecureRandom()); final SSLEngine sslEngine = sslContext.createSSLEngine(); @@ -316,33 +322,39 @@ private boolean saveNewRootCAKeypair() { if (!configDao.update(rootCAPrivateKey.key(), rootCAPrivateKey.category(), CertUtils.privateKeyToPem(keyPair.getPrivate()))) { logger.error("Failed to save RootCA private key"); } + caKeyPair = keyPair; } catch (final NoSuchProviderException | NoSuchAlgorithmException | IOException e) { logger.error("Failed to generate/save RootCA private/public keys due to exception:", e); } - return loadRootCAKeyPair(); + return caKeyPair != null && caKeyPair.getPrivate() != null && caKeyPair.getPublic() != null; } - private boolean saveNewRootCACertificate() { + boolean saveNewRootCACertificate() { if (caKeyPair == null) { throw new CloudRuntimeException("Cannot issue self-signed root CA certificate as CA keypair is not initialized"); } try { logger.debug("Generating root CA certificate"); - final X509Certificate rootCaCertificate = CertUtils.generateV3Certificate( + final X509Certificate generatedCACert = CertUtils.generateV3Certificate( null, caKeyPair, caKeyPair.getPublic(), rootCAIssuerDN.value(), CAManager.CertSignatureAlgorithm.value(), getCaValidityDays(), null, null); - if (!configDao.update(rootCACertificate.key(), rootCACertificate.category(), CertUtils.x509CertificateToPem(rootCaCertificate))) { + if (!configDao.update(rootCACertificate.key(), rootCACertificate.category(), CertUtils.x509CertificateToPem(generatedCACert))) { logger.error("Failed to update RootCA public/x509 certificate"); } + caCertificates = new ArrayList<>(java.util.Collections.singletonList(generatedCACert)); + caCertificate = generatedCACert; } catch (final CertificateException | NoSuchAlgorithmException | NoSuchProviderException | SignatureException | InvalidKeyException | OperatorCreationException | IOException e) { logger.error("Failed to generate RootCA certificate from private/public keys due to exception:", e); return false; } - return loadRootCACertificate(); + return caCertificate != null; } private boolean loadRootCAKeyPair() { + if (caKeyPair != null) { + return true; + } if (StringUtils.isAnyEmpty(rootCAPublicKey.value(), rootCAPrivateKey.value())) { return false; } @@ -355,14 +367,35 @@ private boolean loadRootCAKeyPair() { return caKeyPair.getPrivate() != null && caKeyPair.getPublic() != null; } - private boolean loadRootCACertificate() { + boolean loadRootCACertificate() { + if (caCertificate != null && CollectionUtils.isNotEmpty(caCertificates)) { + return true; + } + caCertificate = null; + caCertificates = null; if (StringUtils.isEmpty(rootCACertificate.value())) { return false; } try { - caCertificate = CertUtils.pemToX509Certificate(rootCACertificate.value()); - caCertificate.verify(caKeyPair.getPublic()); - } catch (final IOException | CertificateException | NoSuchAlgorithmException | InvalidKeyException | SignatureException | NoSuchProviderException e) { + final List loadedCerts = CertUtils.pemToX509Certificates(rootCACertificate.value()); + if (CollectionUtils.isEmpty(loadedCerts)) { + logger.error("No certificates found in ca.plugin.root.ca.certificate"); + return false; + } + final X509Certificate loadedCACert = loadedCerts.get(0); + + // Verify key ownership without enforcing self-signature + if (!loadedCACert.getPublicKey().equals(caKeyPair.getPublic())) { + logger.error("The public key in the CA certificate does not match the configured CA public key"); + return false; + } + + if (loadedCerts.size() > 1) { + logger.info("Loaded CA certificate chain with {} certificate(s)", loadedCerts.size()); + } + caCertificates = loadedCerts; + caCertificate = loadedCACert; + } catch (final IOException | CertificateException e) { logger.error("Failed to load saved RootCA certificate due to exception:", e); return false; } @@ -389,9 +422,15 @@ private boolean loadManagementKeyStore() { try { managementKeyStore = KeyStore.getInstance("JKS"); managementKeyStore.load(null, null); - managementKeyStore.setCertificateEntry(caAlias, caCertificate); + int caIndex = 0; + for (final X509Certificate cert : caCertificates) { + managementKeyStore.setCertificateEntry(caAlias + "-" + caIndex++, cert); + } + final List fullChain = new ArrayList<>(); + fullChain.add(serverCertificate.getClientCertificate()); + fullChain.addAll(caCertificates); managementKeyStore.setKeyEntry(managementAlias, serverCertificate.getPrivateKey(), getKeyStorePassphrase(), - new X509Certificate[]{serverCertificate.getClientCertificate(), caCertificate}); + fullChain.toArray(new X509Certificate[0])); } catch (final CertificateException | NoSuchAlgorithmException | KeyStoreException | IOException e) { logger.error("Failed to load root CA management-server keystore due to exception: ", e); return false; @@ -421,14 +460,63 @@ protected void addConfiguredManagementIp(List ipList) { } + private static void validatePrivateKeyPem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + CertUtils.pemToPrivateKey(value); + } catch (InvalidKeySpecException | IOException e) { + throw new IllegalArgumentException( + "ca.plugin.root.private.key is not a valid PEM private key: " + e.getMessage()); + } + } + + private static void validatePublicKeyPem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + CertUtils.pemToPublicKey(value); + } catch (InvalidKeySpecException | IOException e) { + throw new IllegalArgumentException( + "ca.plugin.root.public.key is not a valid PEM public key: " + e.getMessage()); + } + } + + static void validateCACertificatePem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + final List certs = CertUtils.pemToX509Certificates(value); + if (CollectionUtils.isEmpty(certs)) { + throw new IllegalArgumentException( + "ca.plugin.root.ca.certificate contains no certificates"); + } + } catch (IOException | CertificateException e) { + throw new IllegalArgumentException( + "ca.plugin.root.ca.certificate is not a valid PEM certificate: " + e.getMessage()); + } + } + private boolean setupCA() { - if (!loadRootCAKeyPair() && !saveNewRootCAKeypair()) { - logger.error("Failed to save and load root CA keypair"); - return false; + if (!loadRootCAKeyPair()) { + if (hasUserProvidedCAKeys()) { + logger.error("Failed to load user-provided CA keys from configuration. " + + "Check that ca.plugin.root.private.key, ca.plugin.root.public.key, and " + + "ca.plugin.root.ca.certificate are all set and in the correct PEM format. " + + "Overwriting with auto-generated keys."); + } + if (!saveNewRootCAKeypair()) { + logger.error("Failed to save and load root CA keypair"); + return false; + } } - if (!loadRootCACertificate() && !saveNewRootCACertificate()) { - logger.error("Failed to save and load root CA certificate"); - return false; + if (!loadRootCACertificate()) { + if (hasUserProvidedCAKeys()) { + logger.error("Failed to load user-provided CA certificate. " + + "Check that ca.plugin.root.ca.certificate is set and in PEM format. " + + "Overwriting with auto-generated certificate."); + } + if (!saveNewRootCACertificate()) { + logger.error("Failed to save and load root CA certificate"); + return false; + } } if (!loadManagementKeyStore()) { logger.error("Failed to check and configure management server keystore"); @@ -437,10 +525,16 @@ private boolean setupCA() { return true; } + private boolean hasUserProvidedCAKeys() { + return StringUtils.isNotEmpty(rootCAPublicKey.value()) + || StringUtils.isNotEmpty(rootCAPrivateKey.value()) + || StringUtils.isNotEmpty(rootCACertificate.value()); + } + @Override public boolean start() { managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value(); - return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore(); + return loadRootCAKeyPair() && loadRootCACertificate() && loadManagementKeyStore(); } @Override diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java index d4ded3023321..714e18c3449f 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java @@ -23,9 +23,11 @@ import java.security.KeyPair; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.List; import org.apache.cloudstack.utils.security.CertUtils; import org.junit.Assert; @@ -63,14 +65,14 @@ public void setUp() throws Exception { @Test public void testAuthNotStrictWithInvalidCert() throws Exception { - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(null, null); } @Test public void testAuthNotStrictWithRevokedCert() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(new CrlVO()); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), caCertificate); @@ -79,7 +81,7 @@ public void testAuthNotStrictWithRevokedCert() throws Exception { @Test public void testAuthNotStrictWithInvalidCertOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), caCertificate); @@ -88,14 +90,14 @@ public void testAuthNotStrictWithInvalidCertOwnership() throws Exception { @Test(expected = CertificateException.class) public void testAuthNotStrictWithDenyExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, false, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, false, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); } @Test public void testAuthNotStrictWithAllowExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), expiredClientCertificate); @@ -103,35 +105,50 @@ public void testAuthNotStrictWithAllowExpiredCertAndOwnership() throws Exception @Test(expected = CertificateException.class) public void testAuthStrictWithInvalidCert() throws Exception { - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(null, null); } @Test(expected = CertificateException.class) public void testAuthStrictWithRevokedCert() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(new CrlVO()); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); } @Test(expected = CertificateException.class) public void testAuthStrictWithInvalidCertOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); } @Test(expected = CertificateException.class) public void testAuthStrictWithDenyExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, false, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, false, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); } + @Test + public void testGetAcceptedIssuersWithChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final List chain = Arrays.asList(caCertificate, rootCert); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager( + clientIp, false, true, certMap, chain, crlDao); + + final X509Certificate[] issuers = trustManager.getAcceptedIssuers(); + Assert.assertEquals(2, issuers.length); + Assert.assertEquals(caCertificate, issuers[0]); + Assert.assertEquals(rootCert, issuers[1]); + } + @Test public void testAuthStrictWithAllowExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); Assert.assertTrue(trustManager.getAcceptedIssuers() != null); Assert.assertTrue(trustManager.getAcceptedIssuers().length == 1); Assert.assertEquals(trustManager.getAcceptedIssuers()[0], caCertificate); diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java index 8311f4d45abc..21f00c66a1df 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -38,6 +39,7 @@ import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.SSLUtils; import org.bouncycastle.asn1.x509.GeneralName; @@ -49,7 +51,6 @@ import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.test.util.ReflectionTestUtils; @RunWith(MockitoJUnitRunner.class) @@ -75,7 +76,7 @@ public void setUp() throws Exception { addField(provider, "caKeyPair", caKeyPair); addField(provider, "caCertificate", caCertificate); - addField(provider, "caKeyPair", caKeyPair); + addField(provider, "caCertificates", Collections.singletonList(caCertificate)); } @After @@ -129,6 +130,46 @@ public void testIssueCertificateWithCsr() throws NoSuchProviderException, Certif certificate.getClientCertificate().verify(caCertificate.getPublicKey()); } + @Test + public void testGetCaCertificateWithChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final KeyPair intermediateKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate intermediateCert = CertUtils.generateV3Certificate(rootCert, rootKeyPair, + intermediateKeyPair.getPublic(), "CN=intermediate", "SHA256withRSA", 365, null, null); + + final List chain = Arrays.asList(intermediateCert, rootCert); + addField(provider, "caKeyPair", intermediateKeyPair); + addField(provider, "caCertificate", intermediateCert); + addField(provider, "caCertificates", chain); + + Assert.assertEquals(2, provider.getCaCertificate().size()); + Assert.assertEquals(intermediateCert, provider.getCaCertificate().get(0)); + Assert.assertEquals(rootCert, provider.getCaCertificate().get(1)); + } + + @Test + public void testIssueCertificateWithoutCsrAndChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final KeyPair intermediateKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate intermediateCert = CertUtils.generateV3Certificate(rootCert, rootKeyPair, + intermediateKeyPair.getPublic(), "CN=intermediate", "SHA256withRSA", 365, null, null); + + addField(provider, "caKeyPair", intermediateKeyPair); + addField(provider, "caCertificate", intermediateCert); + addField(provider, "caCertificates", Arrays.asList(intermediateCert, rootCert)); + + final Certificate certificate = provider.issueCertificate(Arrays.asList("domain1.com"), null, 1); + Assert.assertNotNull(certificate); + Assert.assertEquals(2, certificate.getCaCertificates().size()); + Assert.assertEquals(intermediateCert, certificate.getCaCertificates().get(0)); + Assert.assertEquals(rootCert, certificate.getCaCertificates().get(1)); + certificate.getClientCertificate().verify(intermediateKeyPair.getPublic()); + } + @Test public void testRevokeCertificate() throws Exception { Assert.assertTrue(provider.revokeCertificate(CertUtils.generateRandomBigInt(), "anyString")); @@ -177,8 +218,8 @@ public void testIsManagementCertificateNoAltNames() { } @Test - public void testIsManagementCertificateNoMatch() { - ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", "cloudstack"); + public void testIsManagementCertificateNoMatch() throws Exception { + addField(provider, "managementCertificateCustomSAN", "cloudstack"); try { X509Certificate certificate = Mockito.mock(X509Certificate.class); List> altNames = new ArrayList<>(); @@ -193,9 +234,9 @@ public void testIsManagementCertificateNoMatch() { } @Test - public void testIsManagementCertificateMatch() { + public void testIsManagementCertificateMatch() throws Exception { String customSAN = "cloudstack"; - ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", customSAN); + addField(provider, "managementCertificateCustomSAN", customSAN); try { X509Certificate certificate = Mockito.mock(X509Certificate.class); List> altNames = new ArrayList<>(); @@ -208,4 +249,58 @@ public void testIsManagementCertificateMatch() { Assert.fail(String.format("Exception occurred: %s", e.getMessage())); } } + + @Test + public void testLoadRootCACertificateWithMismatchedCert() throws Exception { + KeyPair otherKeyPair = CertUtils.generateRandomKeyPair(1024); + X509Certificate mismatchedCert = CertUtils.generateV3Certificate(null, otherKeyPair, otherKeyPair.getPublic(), "CN=other", "SHA256withRSA", 365, null, null); + String mismatchedPem = CertUtils.x509CertificateToPem(mismatchedCert); + + ConfigKey mockCertKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockCertKey.value()).thenReturn(mismatchedPem); + addField(provider, "rootCACertificate", mockCertKey); + + addField(provider, "caCertificate", null); + addField(provider, "caCertificates", null); + + Boolean result = provider.loadRootCACertificate(); + Assert.assertFalse(result); + Assert.assertNull(provider.getCaCertificate()); + } + + @Test + public void testSaveNewRootCACertificateWithStaleCache() throws Exception { + ConfigurationDao configDao = Mockito.mock(ConfigurationDao.class); + addField(provider, "configDao", configDao); + + ConfigKey mockCertKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockCertKey.key()).thenReturn("ca.plugin.root.ca.certificate"); + Mockito.when(mockCertKey.category()).thenReturn("Hidden"); + addField(provider, "rootCACertificate", mockCertKey); + + ConfigKey mockIssuerKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockIssuerKey.value()).thenReturn("CN=ca.cloudstack.apache.org"); + addField(provider, "rootCAIssuerDN", mockIssuerKey); + + addField(provider, "caCertificate", null); + addField(provider, "caCertificates", null); + + Mockito.when(configDao.update(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(true); + + Boolean result = provider.saveNewRootCACertificate(); + Assert.assertTrue(result); + Assert.assertNotNull(provider.getCaCertificate()); + Assert.assertEquals(1, provider.getCaCertificate().size()); + } + + @Test + public void testValidateCACertificatePem() throws Exception { + String truncatedPem = "-----BEGIN CERTIFICATE-----\nMIICxTCCAa0CAQAw\n"; + try { + RootCAProvider.validateCACertificatePem(truncatedPem); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("is not a valid PEM certificate")); + } + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 0cec0df66182..e5c89c5de1c5 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -17,12 +17,9 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; -import com.cloud.user.Account; - import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -30,36 +27,41 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.api.response.QuotaStatementItemResponse; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaBalance", responseObject = QuotaStatementItemResponse.class, description = "Create a quota balance statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, +@APICommand(name = "quotaBalance", responseObject = QuotaBalanceResponse.class, description = "Create Quota balance statements for the provided Accounts or Projects.", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaBalanceCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Name of the Account for which balance statements will be generated. " + + "Deprecated, please use 'accountid' instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "ID of the domain which the Account identified by 'account' belongs to." + + "Deprecated, please use 'accountid' instead.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End of the period of the Quota balance." + - ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "Date of the last Quota balance to be returned. Must be informed together with the " + + "startdate parameter, and can not be before startdate. " + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start of the period of the Quota balance. " + + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Date of the first Quota balance to be returned. Must be before today. " + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which balance statements will be generated.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which balance statements will be generated. Can not be specified with accountid.") + private Long projectId; + @Inject QuotaResponseBuilder _responseBuilder; @@ -88,48 +90,32 @@ public void setDomainId(Long domainId) { } public Date getEndDate() { - if (endDate == null){ - return null; - } - else{ - return _responseBuilder.startOfNextDay(new Date(endDate.getTime())); - } + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; - } - Account account = _accountService.getActiveAccountByName(accountName, domainId); - if (account != null) { - return account.getAccountId(); + if (ObjectUtils.allNull(accountId, accountName, domainId, projectId)) { + return -1; } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaBalance(this); - - QuotaBalanceResponse response; - if (endDate == null) { - response = _responseBuilder.createQuotaLastBalanceResponse(quotaUsage, getStartDate()); - } else { - response = _responseBuilder.createQuotaBalanceResponse(quotaUsage, getStartDate(), new Date(endDate.getTime())); - } + QuotaBalanceResponse response = _responseBuilder.createQuotaBalanceResponse(this); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index d3bd3868ed16..bfe26a9f4250 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; @@ -28,24 +27,25 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementItemResponse; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; -import com.cloud.user.Account; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a Quota statement for the provided Account, Project, or Domain.", + since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, + description = "Name of the Account for which the Quota statement will be generated. Deprecated, please use accountid instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "ID of the Domain for which the Quota statement will be generated. May be used individually or with account.") private Long domainId; @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End of the period of the Quota statement. " + @@ -56,15 +56,25 @@ public class QuotaStatementCmd extends BaseCmd { ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; - @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") + @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, + description = "Consider only Quota usage records for the specified usage type in the statement.") private Integer usageType; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "ID of the Account for which the Quota statement will be generated. Can not be specified with projectid.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "ID of the Project for which the Quota statement will be generated. Can not be specified with accountid.", since = "4.23.0") + private Long projectId; + + @Parameter(name = ApiConstants.SHOW_RESOURCES, type = CommandType.BOOLEAN, description = "List the resources of each Quota type in the period.", since = "4.23.0") + private boolean showResources; + @Inject - private QuotaResponseBuilder _responseBuilder; + QuotaResponseBuilder responseBuilder; public Long getAccountId() { return accountId; @@ -99,43 +109,47 @@ public void setDomainId(Long domainId) { } public Date getEndDate() { - return _responseBuilder.startOfNextDay(endDate == null ? new Date() : new Date(endDate.getTime())); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; + } + + public boolean isShowResources() { + return showResources; + } + + public void setShowResources(boolean showResources) { + this.showResources = showResources; + } + + public Long getProjectId() { + return projectId; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; } - Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId); - if (activeAccountByName != null) { - return activeAccountByName.getAccountId(); - } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaUsage(this); - - QuotaStatementResponse response = _responseBuilder.createQuotaStatementResponse(quotaUsage); - response.setStartDate(startDate == null ? null : new Date(startDate.getTime())); - response.setEndDate(endDate == null ? null : new Date(endDate.getTime())); - + QuotaStatementResponse response = responseBuilder.createQuotaStatementResponse(this); + response.setStartDate(startDate); + response.setEndDate(endDate); response.setResponseName(getCommandName()); setResponseObject(response); } - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java index 625f4829c673..714066b048ad 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java @@ -17,137 +17,65 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; import java.util.Date; import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import com.cloud.serializer.Param; public class QuotaBalanceResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; - - @SerializedName("startquota") - @Param(description = "Quota started with") - private BigDecimal startQuota; - - @SerializedName("endquota") - @Param(description = "Quota by end of this period") - private BigDecimal endQuota; - - @SerializedName("credits") - @Param(description = "List of credits made during this period") - private List credits = null; - - @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; - - @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; - - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Balance's currency.") private String currency; - public QuotaBalanceResponse() { - super(); - credits = new ArrayList(); - } - - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } + @SerializedName(ApiConstants.DATE) + @Param(description = "Balance's date.") + private Date date; - public void setAccountName(String accountName) { - this.accountName = accountName; - } + @SerializedName(ApiConstants.BALANCE) + @Param(description = "Balance's value.") + private BigDecimal balance; - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } + @SerializedName(ApiConstants.BALANCES) + @Param(description = "List of balances in the period.") + private List balances; - public BigDecimal getStartQuota() { - return startQuota; - } - - public void setStartQuota(BigDecimal startQuota) { - this.startQuota = startQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public BigDecimal getEndQuota() { - return endQuota; - } - - public void setEndQuota(BigDecimal endQuota) { - this.endQuota = endQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public List getCredits() { - return credits; - } - - public void setCredits(List credits) { - this.credits = credits; + public QuotaBalanceResponse() { + super("balance"); } - public void addCredits(QuotaBalanceVO credit) { - QuotaCreditsResponse cr = new QuotaCreditsResponse(); - cr.setCredit(credit.getCreditBalance()); - cr.setCreditedOn(credit.getUpdatedOn()); - credits.add(0, cr); + public QuotaBalanceResponse(Date date, BigDecimal balance) { + super("balance"); + this.date = date; + this.balance = balance; } - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + public void setCurrency(String currency) { + this.currency = currency; } - public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + public void setBalances(List balances) { + this.balances = balances; } - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + public String getCurrency() { + return currency; } - public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + public List getBalances() { + return balances; } - public String getCurrency() { - return currency; + public Date getDate() { + return date; } - public void setCurrency(String currency) { - this.currency = currency; + public BigDecimal getBalance() { + return balance; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 177fb00d4b55..f0390bf626d0 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -29,10 +29,8 @@ import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import java.util.Date; import java.util.List; @@ -49,18 +47,12 @@ public interface QuotaResponseBuilder { boolean isUserAllowedToSeeActivationRules(User user); - QuotaStatementResponse createQuotaStatementResponse(List quotaUsage); + QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); - QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); + QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd); Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); - QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - - List getQuotaUsage(QuotaStatementCmd cmd); - - List getQuotaBalance(QuotaBalanceCmd cmd); - QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); List listQuotaEmailTemplates(QuotaEmailTemplateListCmd cmd); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index d1992d4c3ad7..231ba777e600 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -21,21 +21,17 @@ import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -43,12 +39,37 @@ import javax.inject.Inject; import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.projects.dao.ProjectDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.SnapshotVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaBalanceCmd; @@ -74,6 +95,7 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.ResourceCounting; import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -93,7 +115,8 @@ import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.cloudstack.quota.vo.QuotaUsageResourceVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections4.CollectionUtils; @@ -105,18 +128,6 @@ import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDao; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; -import com.cloud.utils.Pair; - @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { protected Logger logger = LogManager.getLogger(getClass()); @@ -139,8 +150,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject - private ProjectDao projectDao; - @Inject private QuotaAccountDao quotaAccountDao; @Inject private DomainDao domainDao; @@ -158,16 +167,25 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; + @Inject + private IPAddressDao ipAddressDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOfferingDao networkOfferingDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao vmTemplateDao; + @Inject + private VolumeDao volumeDao; - private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; - private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class, ResourceCounting.class}; - protected void checkActivationRulesAllowed(String activationRule) { - if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) { - throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration."); - } - } + private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); @Override public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) { @@ -308,166 +326,225 @@ public boolean isUserAllowedToSeeActivationRules(User user) { } @Override - public QuotaBalanceResponse createQuotaBalanceResponse(List quotaBalance, Date startDate, Date endDate) { - if (quotaBalance == null || quotaBalance.isEmpty()) { - throw new InvalidParameterValueException("The request period does not contain balance entries."); - } - Collections.sort(quotaBalance, new Comparator() { - @Override - public int compare(QuotaBalanceVO o1, QuotaBalanceVO o2) { - o1 = o1 == null ? new QuotaBalanceVO() : o1; - o2 = o2 == null ? new QuotaBalanceVO() : o2; - return o2.getUpdatedOn().compareTo(o1.getUpdatedOn()); // desc - } - }); - - boolean have_balance_entries = false; - //check that there is at least one balance entry - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (entry.isBalanceEntry()) { - have_balance_entries = true; - break; - } + public QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd) { + List quotaBalances = _quotaService.listQuotaBalancesForAccount(cmd.getEntityOwnerId(), cmd.getStartDate(), cmd.getEndDate()); + + List balances = quotaBalances.stream() + .map(balance -> new QuotaBalanceResponse(balance.getUpdatedOn(), balance.getCreditBalance())) + .collect(Collectors.toList()); + + QuotaBalanceResponse response = new QuotaBalanceResponse(); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setBalances(balances); + + return response; + } + + @Override + public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd) { + Long accountId = getAccountIdForQuotaStatement(cmd); + Long domainId = getDomainIdForQuotaStatement(cmd, accountId); + List quotaUsages = _quotaService.getQuotaUsage(accountId, null, domainId, cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); + + logger.debug("Creating quota statement from [{}] usage records for parameters [{}].", quotaUsages.size(), + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "accountId", "projectId", "domainId", "startDate", "endDate", "usageType", "showResources")); + createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(quotaUsages, cmd.getUsageType()); + + Map> recordsPerUsageTypes = quotaUsages.stream() + .sorted(Comparator.comparingInt(QuotaUsageJoinVO::getUsageType)) + .collect(Collectors.groupingBy(QuotaUsageJoinVO::getUsageType)); + + List items = new ArrayList<>(); + recordsPerUsageTypes.forEach((key, value) -> items.add(createStatementItem(key, value, cmd.isShowResources()))); + + QuotaStatementResponse statement = new QuotaStatementResponse(); + statement.setLineItem(items); + statement.setTotalQuota(items.stream().map(QuotaStatementItemResponse::getQuotaUsed).reduce(BigDecimal.ZERO, BigDecimal::add)); + statement.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + statement.setObjectName("statement"); + + if (accountId != null) { + Account account = _accountDao.findByIdIncludingRemoved(accountId); + statement.setAccountId(account.getUuid()); + statement.setAccountName(account.getAccountName()); + domainId = account.getDomainId(); } - //if last entry is a credit deposit then remove that as that is already - //accounted for in the starting balance after that entry, note the sort is desc - if (have_balance_entries) { - ListIterator li = quotaBalance.listIterator(quotaBalance.size()); - // Iterate in reverse. - while (li.hasPrevious()) { - QuotaBalanceVO entry = li.previous(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - li.remove(); - } else { - break; - } - } + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + statement.setDomainId(domain.getUuid()); } - int quota_activity = quotaBalance.size(); - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - boolean consecutive = true; - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: All Credit Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - if (consecutive) { - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.addCredits(entry); - it.remove(); - } else { - consecutive = false; - } + return statement; + } + + protected Long getAccountIdForQuotaStatement(QuotaStatementCmd cmd) { + if (Account.Type.NORMAL.equals(CallContext.current().getCallingAccount().getType())) { + logger.debug("Limiting the Quota statement for the calling Account, as they are a User Account."); + return CallContext.current().getCallingAccountId(); } - if (quota_activity > 0 && quotaBalance.size() > 0) { - // order is desc last item is the start item - QuotaBalanceVO startItem = quotaBalance.get(quotaBalance.size() - 1); - QuotaBalanceVO endItem = quotaBalance.get(0); - resp.setStartDate(startDate); - resp.setStartQuota(startItem.getCreditBalance()); - resp.setEndDate(endDate); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Start Entry=" + startItem); - logger.debug("createQuotaBalanceResponse: End Entry=" + endItem); - } - resp.setEndQuota(endItem.getCreditBalance().add(lastCredits)); - } else if (quota_activity > 0) { - // order is desc last item is the start item - resp.setStartDate(startDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndDate(endDate); - resp.setEndQuota(new BigDecimal(0).add(lastCredits)); - } else { - resp.setStartDate(startDate); - resp.setEndDate(endDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndQuota(new BigDecimal(0)); - } - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; + long accountId = cmd.getEntityOwnerId(); + if (accountId != -1) { + return accountId; + } + + if (cmd.getDomainId() == null) { + logger.debug("Limiting the Quota statement for the calling Account, as 'domainid' was not informed."); + return CallContext.current().getCallingAccountId(); + } + + logger.debug("Allowing admin/domain admin to generate the Quota statement for the provided Domain."); + return null; } - @Override - public QuotaStatementResponse createQuotaStatementResponse(final List quotaUsage) { - if (quotaUsage == null || quotaUsage.isEmpty()) { - throw new InvalidParameterValueException("There is no usage data found for period mentioned."); + protected Long getDomainIdForQuotaStatement(QuotaStatementCmd cmd, Long accountId) { + if (accountId != null) { + logger.debug("Quota statement is already limited to Account [{}].", accountId); + Account account = _accountDao.findByIdIncludingRemoved(accountId); + return account.getDomainId(); } - QuotaStatementResponse statement = new QuotaStatementResponse(); + Long domainId = cmd.getDomainId(); + if (domainId != null) { + return domainId; + } + + logger.debug("Limiting the Quota statement for the calling Account's Domain."); + return CallContext.current().getCallingAccount().getDomainId(); + } - HashMap quotaTariffMap = new HashMap(); - Collection result = QuotaTypes.listQuotaTypes().values(); + protected void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(List quotaUsages, Integer usageType) { + if (usageType != null) { + logger.debug("As the usage type [{}] was informed as parameter of the API quotaStatement, we will not create dummy records.", usageType); + return; - for (QuotaTypes quotaTariff : result) { - quotaTariffMap.put(quotaTariff.getQuotaType(), quotaTariff); - // add dummy record for each usage type - QuotaUsageVO dummy = new QuotaUsageVO(quotaUsage.get(0)); - dummy.setUsageType(quotaTariff.getQuotaType()); - dummy.setQuotaUsed(new BigDecimal(0)); - quotaUsage.add(dummy); } - if (logger.isDebugEnabled()) { - logger.debug( - "createQuotaStatementResponse Type=" + quotaUsage.get(0).getUsageType() + " usage=" + quotaUsage.get(0).getQuotaUsed().setScale(2, RoundingMode.HALF_EVEN) - + " rec.id=" + quotaUsage.get(0).getUsageItemId() + " SD=" + quotaUsage.get(0).getStartDate() + " ED=" + quotaUsage.get(0).getEndDate()); + for (Integer quotaType : QuotaTypes.listQuotaTypes().keySet()) { + QuotaUsageJoinVO dummy = new QuotaUsageJoinVO(); + dummy.setUsageType(quotaType); + dummy.setQuotaUsed(BigDecimal.ZERO); + quotaUsages.add(dummy); } + } + + protected QuotaStatementItemResponse createStatementItem(int usageType, List usageRecords, boolean showResources) { + QuotaUsageJoinVO firstRecord = usageRecords.get(0); + int type = firstRecord.getUsageType(); + + QuotaTypes quotaType = QuotaTypes.listQuotaTypes().get(type); + + QuotaStatementItemResponse item = new QuotaStatementItemResponse(type); + item.setQuotaUsed(usageRecords.stream().map(QuotaUsageJoinVO::getQuotaUsed).filter(Objects::nonNull).reduce(BigDecimal.ZERO, BigDecimal::add)); + item.setUsageUnit(quotaType.getQuotaUnit()); + item.setUsageName(quotaType.getQuotaName()); + + setStatementItemResources(item, usageType, usageRecords, showResources); + return item; + } + + protected void setStatementItemResources(QuotaStatementItemResponse statementItem, int usageType, List quotaUsageRecords, boolean showResources) { + if (!showResources) { + return; + } + + List itemDetails = new ArrayList<>(); + + Map quotaUsagesValuesAggregatedById = quotaUsageRecords + .stream() + .filter(quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType) != null) + .collect(Collectors.groupingBy( + quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType), + Collectors.reducing(new BigDecimal(0), QuotaUsageJoinVO::getQuotaUsed, BigDecimal::add) + )); + + for (Map.Entry entry : quotaUsagesValuesAggregatedById.entrySet()) { + QuotaStatementItemResourceResponse detail = new QuotaStatementItemResourceResponse(); + + detail.setQuotaUsed(entry.getValue()); + + QuotaUsageResourceVO resource = getResourceFromIdAndType(entry.getKey(), usageType); + if (resource != null) { + detail.setResourceId(resource.getUuid()); + detail.setDisplayName(resource.getName()); + detail.setRemoved(resource.isRemoved()); + } else { + detail.setDisplayName(""); - Collections.sort(quotaUsage, new Comparator() { - @Override - public int compare(QuotaUsageVO o1, QuotaUsageVO o2) { - if (o1.getUsageType() == o2.getUsageType()) { - return 0; - } - return o1.getUsageType() < o2.getUsageType() ? -1 : 1; - } - }); - - List items = new ArrayList(); - QuotaStatementItemResponse lineitem; - int type = -1; - BigDecimal usage = new BigDecimal(0); - BigDecimal totalUsage = new BigDecimal(0); - quotaUsage.add(new QuotaUsageVO());// boundary - QuotaUsageVO prev = quotaUsage.get(0); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaStatementResponse record count=" + quotaUsage.size()); - } - for (final QuotaUsageVO quotaRecord : quotaUsage) { - if (type != quotaRecord.getUsageType()) { - if (type != -1) { - lineitem = new QuotaStatementItemResponse(type); - lineitem.setQuotaUsed(usage); - lineitem.setAccountId(prev.getAccountId()); - lineitem.setDomainId(prev.getDomainId()); - lineitem.setUsageUnit(quotaTariffMap.get(type).getQuotaUnit()); - lineitem.setUsageName(quotaTariffMap.get(type).getQuotaName()); - lineitem.setObjectName("quotausage"); - items.add(lineitem); - totalUsage = totalUsage.add(usage); - usage = new BigDecimal(0); - } - type = quotaRecord.getUsageType(); } - prev = quotaRecord; - usage = usage.add(quotaRecord.getQuotaUsed()); + itemDetails.add(detail); + } + statementItem.setResources(itemDetails); + } + + protected Long getResourceIdByUsageType(QuotaUsageJoinVO quotaUsageJoinVo, int usageType) { + switch (usageType) { + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + return quotaUsageJoinVo.getNetworkId(); + case QuotaTypes.NETWORK_OFFERING: + return quotaUsageJoinVo.getOfferingId(); + default: + return quotaUsageJoinVo.getResourceId(); } + } - statement.setLineItem(items); - statement.setTotalQuota(totalUsage); - statement.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - statement.setObjectName("statement"); - return statement; + protected QuotaUsageResourceVO getResourceFromIdAndType(long resourceId, int usageType) { + switch (usageType) { + case QuotaTypes.ALLOCATED_VM: + case QuotaTypes.RUNNING_VM: + VMInstanceVO vmInstance = vmInstanceDao.findByIdIncludingRemoved(resourceId); + if (vmInstance != null) { + return new QuotaUsageResourceVO(vmInstance.getUuid(), vmInstance.getHostName(), vmInstance.getRemoved()); + } + break; + case QuotaTypes.VOLUME: + case QuotaTypes.VOLUME_SECONDARY: + case QuotaTypes.VM_DISK_BYTES_READ: + case QuotaTypes.VM_DISK_BYTES_WRITE: + case QuotaTypes.VM_DISK_IO_READ: + case QuotaTypes.VM_DISK_IO_WRITE: + VolumeVO volume = volumeDao.findByIdIncludingRemoved(resourceId); + if (volume != null) { + return new QuotaUsageResourceVO(volume.getUuid(), volume.getName(), volume.getRemoved()); + } + break; + case QuotaTypes.VM_SNAPSHOT_ON_PRIMARY: + case QuotaTypes.VM_SNAPSHOT: + case QuotaTypes.SNAPSHOT: + SnapshotVO snapshot = snapshotDao.findByIdIncludingRemoved(resourceId); + if (snapshot != null) { + return new QuotaUsageResourceVO(snapshot.getUuid(), snapshot.getName(), snapshot.getRemoved()); + } + break; + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + NetworkVO network = networkDao.findByIdIncludingRemoved(resourceId); + if (network != null) { + return new QuotaUsageResourceVO(network.getUuid(), network.getName(), network.getRemoved()); + } + break; + case QuotaTypes.TEMPLATE: + case QuotaTypes.ISO: + VMTemplateVO vmTemplate = vmTemplateDao.findByIdIncludingRemoved(resourceId); + if (vmTemplate != null) { + return new QuotaUsageResourceVO(vmTemplate.getUuid(), vmTemplate.getName(), vmTemplate.getRemoved()); + } + break; + case QuotaTypes.NETWORK_OFFERING: + NetworkOfferingVO networkOffering = networkOfferingDao.findByIdIncludingRemoved(resourceId); + if (networkOffering != null) { + return new QuotaUsageResourceVO(networkOffering.getUuid(), networkOffering.getName(), networkOffering.getRemoved()); + } + break; + case QuotaTypes.IP_ADDRESS: + IPAddressVO ipAddress = ipAddressDao.findByIdIncludingRemoved(resourceId); + if (ipAddress != null) { + return new QuotaUsageResourceVO(ipAddress.getUuid(), ipAddress.getName(), ipAddress.getRemoved()); + } + break; + } + return null; } @Override @@ -501,6 +578,7 @@ public QuotaTariffVO updateQuotaTariffPlan(QuotaTariffUpdateCmd cmd) { Integer position = cmd.getPosition(); warnQuotaTariffUpdateDeprecatedFields(cmd); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); @@ -508,8 +586,6 @@ public QuotaTariffVO updateQuotaTariffPlan(QuotaTariffUpdateCmd cmd) { throw new InvalidParameterValueException(String.format("There is no quota tariffs with name [%s].", name)); } - checkActivationRulesAllowed(activationRule); - Date currentQuotaTariffStartDate = currentQuotaTariff.getEffectiveOn(); currentQuotaTariff.setRemoved(now); @@ -524,14 +600,14 @@ public QuotaTariffVO updateQuotaTariffPlan(QuotaTariffUpdateCmd cmd) { } protected void warnQuotaTariffUpdateDeprecatedFields(QuotaTariffUpdateCmd cmd) { - String warnMessage = "The parameter 's%s' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; + String warnMessage = "The parameter '{}' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; if (cmd.getStartDate() != null) { - logger.warn(String.format(warnMessage,"startdate")); + logger.warn(warnMessage, "startdate"); } if (cmd.getUsageType() != null) { - logger.warn(String.format(warnMessage,"usagetype")); + logger.warn(warnMessage, "usagetype"); } } @@ -630,7 +706,7 @@ public QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Doubl throw new InvalidParameterValueException("Account does not exist with account id " + accountId); } final boolean lockAccountEnforcement = "true".equalsIgnoreCase(QuotaConfig.QuotaEnableEnforcement.value()); - final BigDecimal currentAccountBalance = _quotaBalanceDao.lastQuotaBalance(accountId, domainId, startOfNextDay(new Date(depositedOn.getTime()))); + final BigDecimal currentAccountBalance = _quotaBalanceDao.getLastQuotaBalance(accountId, domainId); logger.debug("Depositing [{}] credits on adjusted date [{}]; current balance is [{}].", amount, DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), depositedOn), currentAccountBalance); // update quota account with the balance @@ -694,39 +770,6 @@ public boolean updateQuotaEmailTemplate(QuotaEmailTemplateUpdateCmd cmd) { return false; } - @Override - public QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate) { - if (quotaBalance == null) { - throw new InvalidParameterValueException("There are no balance entries on or before the requested date."); - } - if (startDate == null) { - startDate = new Date(); - } - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - for (QuotaBalanceVO entry : quotaBalance) { - logger.debug("createQuotaLastBalanceResponse Date={} balance={} credit={}", - DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), entry.getUpdatedOn()), - entry.getCreditBalance(), entry.getCreditsId()); - - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.setStartQuota(lastCredits); - resp.setStartDate(startDate); - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; - } - - @Override - public List getQuotaUsage(QuotaStatementCmd cmd) { - return _quotaService.getQuotaUsage(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); - } - - @Override - public List getQuotaBalance(QuotaBalanceCmd cmd) { - return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); - } @Override public Date startOfNextDay(Date date) { LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); @@ -758,14 +801,14 @@ public QuotaTariffVO createQuotaTariff(QuotaTariffCreateCmd cmd) { String activationRule = cmd.getActivationRule(); Integer position = ObjectUtils.defaultIfNull(cmd.getPosition(), 1); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); + QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); if (currentQuotaTariff != null) { throw new InvalidParameterValueException(String.format("A quota tariff with name [%s] already exist.", name)); } - checkActivationRulesAllowed(activationRule); - if (startDate.compareTo(now) < 0) { throw new InvalidParameterValueException(String.format("The value passed as Quota tariff's start date is in the past: [%s]. " + "Please, inform a date in the future or do not pass the parameter to use the current date and time.", startDate)); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java new file mode 100644 index 000000000000..3e052f733391 --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java @@ -0,0 +1,61 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import java.math.BigDecimal; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class QuotaStatementItemResourceResponse extends BaseResponse { + + @SerializedName(ApiConstants.QUOTA_CONSUMED) + @Param(description = "Quota consumed.") + private BigDecimal quotaUsed; + + @SerializedName(ApiConstants.RESOURCE_ID) + @Param(description = "Resources's ID.") + private String resourceId; + + @SerializedName(ApiConstants.DISPLAY_NAME) + @Param(description = "Resource's display name.") + private String displayName; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "Indicates whether the resource is removed or active.") + private boolean removed; + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setRemoved(boolean removed) { + this.removed = removed; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java index c370d82b3cc4..0747c5a9172d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java @@ -17,72 +17,41 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; +import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; public class QuotaStatementItemResponse extends BaseResponse { - @SerializedName("type") - @Param(description = "Usage type") + @SerializedName(ApiConstants.TYPE) + @Param(description = "Usage type.") private int usageType; - @SerializedName("accountid") - @Param(description = "Account id") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain id") - private Long domainId; - - @SerializedName("name") - @Param(description = "Usage type name") + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Usage type.") private String usageName; - @SerializedName("unit") - @Param(description = "Usage unit") + @SerializedName(ApiConstants.UNIT) + @Param(description = "Unit of the Usage type.") private String usageUnit; - @SerializedName("quota") - @Param(description = "Quota consumed") + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Quota consumed.") private BigDecimal quotaUsed; + @SerializedName(ApiConstants.RESOURCES) + @Param(description = "Item's resources.") + private List resources; + public QuotaStatementItemResponse(final int usageType) { this.usageType = usageType; } - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - public String getUsageName() { return usageName; } @@ -112,7 +81,15 @@ public BigDecimal getQuotaUsed() { } public void setQuotaUsed(BigDecimal quotaUsed) { - this.quotaUsed = quotaUsed.setScale(2, RoundingMode.HALF_EVEN); + this.quotaUsed = quotaUsed; + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + this.resources = resources; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java index 0a7ba4dbeb94..81cb1011182d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java @@ -18,56 +18,56 @@ import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import java.util.List; public class QuotaStatementResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "ID of the Account.") + private String accountId; - @SerializedName("account") - @Param(description = "Account name") + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "Name of the Account.") private String accountName; - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "ID of the Domain.") + private String domainId; - @SerializedName("quotausage") - @Param(description = "List of quota usage under various types", responseObject = QuotaStatementItemResponse.class) + @SerializedName(ApiConstants.QUOTA_USAGE) + @Param(description = "List of Quota usage under various types.", responseObject = QuotaStatementItemResponse.class) private List lineItem; - @SerializedName("totalquota") - @Param(description = "Total quota used during this period") + @SerializedName(ApiConstants.TOTAL_QUOTA) + @Param(description = "Total Quota consumed during this period.") private BigDecimal totalQuota; - @SerializedName("startdate") - @Param(description = "Start date") + @SerializedName(ApiConstants.START_DATE) + @Param(description = "Start date of the Quota statement.") private Date startDate = null; - @SerializedName("enddate") - @Param(description = "End date") + @SerializedName(ApiConstants.END_DATE) + @Param(description = "End date of the Quota statement.") private Date endDate = null; - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Currency of the Quota statement.") private String currency; public QuotaStatementResponse() { super(); } - public Long getAccountId() { + public String getAccountId() { return accountId; } - public void setAccountId(Long accountId) { + public void setAccountId(String accountId) { this.accountId = accountId; } @@ -79,45 +79,36 @@ public void setAccountName(String accountName) { this.accountName = accountName; } - public Long getDomainId() { + public String getDomainId() { return domainId; } - public void setDomainId(Long domainId) { + public void setDomainId(String domainId) { this.domainId = domainId; } - public List getLineItem() { - return lineItem; - } - public void setLineItem(List lineItem) { this.lineItem = lineItem; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - - public BigDecimal getTotalQuota() { - return totalQuota; + this.endDate = endDate; } public void setTotalQuota(BigDecimal totalQuota) { - this.totalQuota = totalQuota.setScale(2, RoundingMode.HALF_EVEN); + this.totalQuota = totalQuota; } public String getCurrency() { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index f6a34e01be8d..478e43d2e203 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -21,16 +21,16 @@ import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import com.cloud.user.AccountVO; import com.cloud.utils.component.PluggableService; public interface QuotaService extends PluggableService { - List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); + List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); - List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); + List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate); void setLockAccount(Long accountId, Boolean state); @@ -40,6 +40,4 @@ public interface QuotaService extends PluggableService { boolean saveQuotaAccount(AccountVO account, BigDecimal aggrUsage, Date endDate); - boolean isJsInterpretationEnabled(); - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index f39153ae0ac6..e1ec2ebb2a60 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -28,6 +28,7 @@ import com.cloud.projects.ProjectManager; import com.cloud.user.AccountService; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -53,23 +54,21 @@ import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; -import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.server.ManagementService; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.db.Filter; @Component public class QuotaServiceImpl extends ManagerBase implements QuotaService, Configurable, QuotaConfig { @@ -81,7 +80,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private QuotaAccountDao _quotaAcc; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageJoinDao quotaUsageJoinDao; @Inject private DomainDao _domainDao; @Inject @@ -95,8 +94,6 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi private TimeZone _usageTimezone; - private boolean jsInterpretationEnabled = false; - public QuotaServiceImpl() { super(); } @@ -108,8 +105,6 @@ public boolean configure(String name, Map params) throws Configu String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT"); _usageTimezone = TimeZone.getTimeZone(timeZoneStr); - jsInterpretationEnabled = ManagementService.JsInterpretationEnabled.value(); - return true; } @@ -157,88 +152,58 @@ public Boolean isQuotaServiceEnabled() { } @Override - public List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate) { - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } + public List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate) { + validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + + if (accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); } + Account account = _accountDao.findByIdIncludingRemoved(accountId); + Long domainId = account.getDomainId(); - startDate = startDate == null ? new Date() : startDate; + if (startDate == null && endDate == null) { + logger.debug("Retrieving last quota balance for {}.", account); + QuotaBalanceVO lastQuotaBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, null); - if (endDate == null) { - // adjust start date to end of day as there is no end date - startDate = _respBldr.startOfNextDay(startDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance1: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", on or before " + startDate); - } - List qbrecords = _quotaBalanceDao.lastQuotaBalanceVO(accountId, domainId, startDate); - if (logger.isDebugEnabled()) { - logger.debug("Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("Incorrect Date there are no quota records before this date " + startDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - if (startDate.before(endDate)) { - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance2: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", between " + startDate - + " and " + endDate); - } - List qbrecords = _quotaBalanceDao.findQuotaBalance(accountId, domainId, startDate, endDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance3: Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("There are no quota records between these dates start date " + startDate + " and end date:" + endDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); + if (lastQuotaBalance == null) { + logger.debug("Did not found a quota balance entry for {}.", account); + return new ArrayList<>(); } + + return List.of(lastQuotaBalance); + } + + if (endDate == null) { + endDate = DateUtils.addDays(new Date(), -1); } + + List quotaBalances = _quotaBalanceDao.listQuotaBalances(accountId, domainId, startDate, endDate); + + if (quotaBalances.isEmpty()) { + logger.info("There are no quota balances for {} between [{}] and [{}].", account, + DateUtil.getOutputString(startDate), DateUtil.getOutputString(endDate)); + } + + return quotaBalances; } - @Override - public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { - // if accountId is not specified, use accountName and domainId - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } + protected void validateStartDateAndEndDateForListQuotaBalancesForAccount(Date startDate, Date endDate) { + if (startDate == null && endDate != null) { + throw new InvalidParameterValueException("Parameter \"enddate\" must be informed together with parameter \"startdate\"."); + } + + Date now = new Date(); + if (startDate != null && startDate.after(now)) { + throw new InvalidParameterValueException("The last balance can be at most from yesterday; therefore, the start date must be before today."); } + if (ObjectUtils.allNotNull(startDate, endDate) && startDate.after(endDate)) { + throw new InvalidParameterValueException("The start date cannot be after the end date."); + } + } + + @Override + public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } @@ -246,7 +211,7 @@ public List getQuotaUsage(Long accountId, String accountName, Long logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", usageType, accountId, domainId, startDate, endDate); - return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, startDate, endDate); + return quotaUsageJoinDao.findQuotaUsage(accountId, domainId, usageType, null, null, null, startDate, endDate, null); } @Override @@ -298,9 +263,4 @@ public void setMinBalance(Long accountId, Double balance) { _quotaAcc.updateQuotaAccount(accountId, acc); } } - - @Override - public boolean isJsInterpretationEnabled() { - return jsInterpretationEnabled; - } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java index adabc694f25a..cd0eeb2c3b48 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java @@ -16,18 +16,15 @@ // under the License. package org.apache.cloudstack.api.command; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import junit.framework.TestCase; @@ -36,31 +33,20 @@ public class QuotaBalanceCmdTest extends TestCase { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder quotaResponseBuilderMock; - @Test - public void testQuotaBalanceCmd() throws NoSuchFieldException, IllegalAccessException { - QuotaBalanceCmd cmd = new QuotaBalanceCmd(); - Field rbField = QuotaBalanceCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + @InjectMocks + @Spy + QuotaBalanceCmd quotaBalanceCmdSpy; - List quotaBalanceVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaBalance(Mockito.any(cmd.getClass()))).thenReturn(quotaBalanceVOList); - Mockito.when(responseBuilder.createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.when(responseBuilder.createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.lenient().when(responseBuilder.startOfNextDay(Mockito.any(Date.class))).thenReturn(new Date()); + @Test + public void executeTestSetResponseObject() { + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + Mockito.doReturn(expected).when(quotaResponseBuilderMock).createQuotaBalanceResponse(Mockito.eq(quotaBalanceCmdSpy)); - // end date not specified - cmd.setStartDate(new Date()); - cmd.setEndDate(null); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class)); - Mockito.verify(responseBuilder, Mockito.times(0)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + quotaBalanceCmdSpy.execute(); - // end date specified - cmd.setEndDate(new Date()); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + Assert.assertEquals(expected, quotaBalanceCmdSpy.getResponseObject()); } + } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java index d6f9f747fa84..e67638db632e 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java @@ -16,38 +16,29 @@ // under the License. package org.apache.cloudstack.api.command; -import junit.framework.TestCase; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) -public class QuotaStatementCmdTest extends TestCase { +public class QuotaStatementCmdTest { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder responseBuilderMock; @Test - public void testQuotaStatementCmd() throws NoSuchFieldException, IllegalAccessException { + public void executeTestVerifyCalls() { QuotaStatementCmd cmd = new QuotaStatementCmd(); cmd.setAccountName("admin"); + cmd.responseBuilder = responseBuilderMock; - Field rbField = QuotaStatementCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + Mockito.doReturn(new QuotaStatementResponse()).when(responseBuilderMock).createQuotaStatementResponse(Mockito.any()); - List quotaUsageVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaUsage(Mockito.eq(cmd))).thenReturn(quotaUsageVOList); - Mockito.when(responseBuilder.createQuotaStatementResponse(Mockito.eq(quotaUsageVOList))).thenReturn(new QuotaStatementResponse()); cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).getQuotaUsage(Mockito.eq(cmd)); + + Mockito.verify(responseBuilderMock).createQuotaStatementResponse(cmd); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index ea88a106b846..0c7610461c15 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -38,10 +40,12 @@ import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; @@ -67,6 +71,7 @@ import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.lang3.time.DateUtils; @@ -245,7 +250,7 @@ public void testAddQuotaCredits() { credit.setCredit(new BigDecimal(amount)); Mockito.when(quotaCreditsDaoMock.saveCredits(Mockito.any(QuotaCreditsVO.class))).thenReturn(credit); - Mockito.when(quotaBalanceDaoMock.lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class))).thenReturn(new BigDecimal(111)); + Mockito.when(quotaBalanceDaoMock.getLastQuotaBalance(Mockito.anyLong(), Mockito.anyLong())).thenReturn(new BigDecimal(111)); Mockito.doReturn(userVoMock).when(quotaResponseBuilderSpy).getCreditorForQuotaCredits(credit); AccountVO account = new AccountVO(); @@ -291,33 +296,6 @@ public void testUpdateQuotaEmailTemplate() { assertTrue(quotaResponseBuilderSpy.updateQuotaEmailTemplate(cmd)); } - @Test - public void testCreateQuotaLastBalanceResponse() { - List quotaBalance = new ArrayList<>(); - // null balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(null, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // empty balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // valid balance test - QuotaBalanceVO entry = new QuotaBalanceVO(); - entry.setAccountId(2L); - entry.setCreditBalance(new BigDecimal(100)); - quotaBalance.add(entry); - quotaBalance.add(entry); - QuotaBalanceResponse resp = quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, null); - assertTrue(resp.getStartQuota().compareTo(new BigDecimal(200)) == 0); - } - @Test public void testStartOfNextDayWithoutParameters() { Date nextDate = quotaResponseBuilderSpy.startOfNextDay(); @@ -914,4 +892,239 @@ public void injectUsageTypeVariablesTestReturnInjectedVariables() { Assert.assertTrue(formattedVariables.containsValue("accountname")); Assert.assertTrue(formattedVariables.containsValue("zonename")); } + + private List getQuotaBalancesForTest() { + List balances = new ArrayList<>(); + + QuotaBalanceVO balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-10.42)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-18.94)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-29.37)); + balances.add(balance); + + return balances; + } + + @Test + public void createQuotaBalancesResponseTestCreateResponse() { + List balances = getQuotaBalancesForTest(); + + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + expected.setObjectName("balance"); + expected.setCurrency("$"); + + Mockito.doReturn(balances).when(quotaServiceMock).listQuotaBalancesForAccount(Mockito.any(), Mockito.any(), Mockito.any()); + QuotaBalanceResponse result = quotaResponseBuilderSpy.createQuotaBalanceResponse(new QuotaBalanceCmd()); + + Assert.assertEquals(expected.getCurrency(), result.getCurrency()); + + for (int i = 0; i < balances.size(); i++) { + Assert.assertEquals(balances.get(i).getUpdatedOn(), result.getBalances().get(i).getDate()); + Assert.assertEquals(balances.get(i).getCreditBalance(), result.getBalances().get(i).getBalance()); + } + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { + List listUsage = new ArrayList<>(); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, 1); + + Assert.assertTrue(listUsage.isEmpty()); + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeIsNullAddDummyForAllQuotaTypes() { + List listUsage = new ArrayList<>(); + listUsage.add(new QuotaUsageJoinVO()); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, null); + + Assert.assertEquals(QuotaTypes.listQuotaTypes().size() + 1, listUsage.size()); + + QuotaTypes.listQuotaTypes().entrySet().forEach(entry -> { + Assert.assertTrue(listUsage.stream().anyMatch(usage -> usage.getUsageType() == entry.getKey() && usage.getQuotaUsed().equals(BigDecimal.ZERO))); + }); + } + + private List getQuotaUsagesForTest() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + List quotaUsages = new ArrayList<>(); + + QuotaUsageJoinVO quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(1l); + quotaUsage.setDomainId(2l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(10)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-01")); + quotaUsage.setEndDate(sdf.parse("2022-01-02")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(4l); + quotaUsage.setDomainId(5l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(null); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-03")); + quotaUsage.setEndDate(sdf.parse("2022-01-04")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(6l); + quotaUsage.setDomainId(7l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(5)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-05")); + quotaUsage.setEndDate(sdf.parse("2022-01-06")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + return quotaUsages; + } + + @Test + public void createStatementItemTestReturnItem() { + List quotaUsages = getQuotaUsagesForTest(); + Mockito.doNothing().when(quotaResponseBuilderSpy).setStatementItemResources(Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyBoolean()); + + QuotaStatementItemResponse result = quotaResponseBuilderSpy.createStatementItem(0, quotaUsages, false); + + QuotaUsageJoinVO expected = quotaUsages.get(0); + QuotaTypes quotaTypeExpected = QuotaTypes.listQuotaTypes().get(expected.getUsageType()); + Assert.assertEquals(BigDecimal.valueOf(15), result.getQuotaUsed()); + Assert.assertEquals(quotaTypeExpected.getQuotaUnit(), result.getUsageUnit()); + Assert.assertEquals(quotaTypeExpected.getQuotaName(), result.getUsageName()); + } + + @Test + public void setStatementItemResourcesTestDoNotShowResourcesDoNothing() { + QuotaStatementItemResponse item = new QuotaStatementItemResponse(1); + + quotaResponseBuilderSpy.setStatementItemResources(item, 0, getQuotaUsagesForTest(), false); + + Assert.assertNull(item.getResources()); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountForNormalUser() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsEntityOwnerIdWhenProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(42L).when(cmd).getEntityOwnerId(); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(42L), result); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountWhenCallerIsAdminAndDomainIsNotProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(null).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminAndDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(10L).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertNull(result); + } + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsAccountDomainIdWhenAccountIdIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + AccountVO account = Mockito.mock(AccountVO.class); + + Mockito.doReturn(account).when(accountDaoMock).findByIdIncludingRemoved(55L); + Mockito.doReturn(77L).when(account).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, 55L); + + Assert.assertEquals(Long.valueOf(77L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsProvidedDomainIdWhenAccountIdIsNull() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(99L).when(cmd).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(Long.valueOf(99L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestFallsBackToCallingAccountDomainIdWhenNeitherAccountNorDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + Account account = Mockito.mock(Account.class); + + Mockito.doReturn(null).when(cmd).getDomainId(); + Mockito.doReturn(123L).when(account).getDomainId(); + Mockito.doReturn(account).when(callContextMock).getCallingAccount(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(123L, result.longValue()); + } + } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 259264f3b0e0..89c48b1cc660 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; @@ -28,9 +29,12 @@ import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.time.DateUtils; import org.joda.time.DateTime; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,7 +46,6 @@ import javax.naming.ConfigurationException; import java.lang.reflect.Field; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -50,6 +53,8 @@ @RunWith(MockitoJUnitRunner.class) public class QuotaServiceImplTest extends TestCase { + @Mock + AccountVO accountVoMock; @Mock AccountDao accountDaoMock; @Mock @@ -63,10 +68,9 @@ public class QuotaServiceImplTest extends TestCase { @Mock QuotaBalanceDao quotaBalanceDao; @Mock - QuotaResponseBuilder respBldr; + QuotaUsageJoinDao quotaUsageJoinDaoMock; @Mock - private AccountVO accountVoMock; - + QuotaResponseBuilder respBldr; @Spy @InjectMocks QuotaServiceImpl quotaServiceImplSpy; @@ -85,9 +89,9 @@ public void setup() throws IllegalAccessException, NoSuchFieldException, Configu quotaAccountDaoField.setAccessible(true); quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); - Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); + Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("quotaUsageJoinDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageJoinDaoMock); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); @@ -109,30 +113,6 @@ public void setup() throws IllegalAccessException, NoSuchFieldException, Configu quotaServiceImplSpy.configure("randomName", null); } - @Test - public void testFindQuotaBalanceVO() { - final long accountId = 2L; - final String accountName = "admin123"; - final long domainId = 1L; - final Date startDate = new DateTime().minusDays(2).toDate(); - final Date endDate = new Date(); - - List records = new ArrayList<>(); - QuotaBalanceVO qb = new QuotaBalanceVO(); - qb.setCreditBalance(new BigDecimal(100)); - qb.setAccountId(accountId); - records.add(qb); - - Mockito.when(respBldr.startOfNextDay(Mockito.any(Date.class))).thenReturn(startDate); - Mockito.when(quotaBalanceDao.findQuotaBalance(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(records); - Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); - - // with enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); - // without enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); - } - @Test public void testGetQuotaUsage() { final long accountId = 2L; @@ -142,7 +122,8 @@ public void testGetQuotaUsage() { final Date endDate = new Date(); quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); - Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); + Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); } @Test @@ -178,4 +159,66 @@ public void testSetMinBalance() { Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsNullAndEndDateIsNotNullThrowsInvalidParameterException() { + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(null, new Date()); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterNowThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, null); + } + + @Test + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestEndDateIsAfterNowDoesNothing() { + Date startDate = DateUtils.addMinutes(new Date(), -1); + Date endDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterEndDateThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), -10); + Date endDate = DateUtils.addMinutes(new Date(), -15); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNullReturnsEmptyList() { + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(null).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNotNullReturnsIt() { + QuotaBalanceVO expected = new QuotaBalanceVO(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertEquals(expected, result.get(0)); + } + + @Test + public void listQuotaBalancesForAccountTestReturnsQuotaBalances() { + List expected = new ArrayList<>(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).listQuotaBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, new Date(), null); + + Assert.assertEquals(expected, result); + } + } diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java index ac840c00be32..3f2d85458d30 100644 --- a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java @@ -48,6 +48,8 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; +import javax.net.ssl.SSLContext; + import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustAllStrategy; import org.apache.http.entity.ContentType; @@ -97,7 +99,9 @@ protected boolean isValidJson(String json) { protected void setHttpClient() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { if (webhook.isSslVerification()) { - httpClient = HttpClients.createDefault(); + httpClient = HttpClients.custom() + .setSSLContext(SSLContext.getDefault()) + .build(); return; } httpClient = HttpClients diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index 061caf1573ff..000000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.23.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java deleted file mode 100644 index 42129944a194..000000000000 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -@Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject - private CapacityManager capacityManager; - - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::toString).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - - if (type == Host.Type.Storage) { - return suitableHosts; - } - String offeringHostTag = offering.getHostTag(); - - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostTag = ObjectUtils.allNotNull(offeringHostTag, templateTag) ? - String.format("%s, %s", offeringHostTag, templateTag) : - ObjectUtils.firstNonNull(offeringHostTag, templateTag); - logger.debug("Looking for hosts in dc [{}], pod [{}], cluster [{}] and complying with host tag(s): [{}]", dcId, podId, clusterId, hostTag); - } else { - logger.debug("Looking for hosts in dc: {} pod: {} cluster: {}", dcId , podId, clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); - - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); - return null; - } - - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } - - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { - if (suitableHosts.size() == returnUpTo) { - break; - } - if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Host %s is in avoid set, skipping this and trying other available hosts", host)); - } - continue; - } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Not using host %s; host has cpu capability? %s, host has capacity? %s", host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second())); - } - continue; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found a suitable host, adding to list: %s", host)); - } - suitableHosts.add(host); - } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } - return new ArrayList<>(); - } - return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } -} diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java deleted file mode 100644 index 538d7157184a..000000000000 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; - -@RunWith(MockitoJUnitRunner.class) -public class RandomAllocatorTest { - - @Mock - HostDao hostDao; - @InjectMocks - RandomAllocator randomAllocator; - - @Test - public void testListHostsByTags() { - Host.Type type = Host.Type.Routing; - Long id = 1L; - String templateTag = "tag1"; - String offeringTag = "tag2"; - HostVO host1 = Mockito.mock(HostVO.class); - HostVO host2 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); - - // No template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Different template tagged host - HostVO host3 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Matching template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - - // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); - - // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - } -} diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java index 940897de3c95..c6c38a398098 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java @@ -106,7 +106,6 @@ public VMTemplateVO create(TemplateProfile profile) { } } - _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); return template; } diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java index 3d79b9efdd13..4e44d8cb7359 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java @@ -41,15 +41,15 @@ public class HypervInvestigator extends AdapterBase implements Investigator { @Override public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM { - Status status = isAgentAlive(host); + Status status = getHostAgentStatus(host); if (status == null) { throw new UnknownVM(); } - return status == Status.Up ? true : null; + return status == Status.Up; } @Override - public Status isAgentAlive(Host agent) { + public Status getHostAgentStatus(Host agent) { if (agent.getHypervisorType() != Hypervisor.HypervisorType.Hyperv) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java index ce9fbe6c232c..da9a0d6e2919 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java @@ -19,10 +19,7 @@ package com.cloud.ha; import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.host.Host; -import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; @@ -34,11 +31,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.ha.HAManager; +import org.apache.cloudstack.kvm.ha.KVMHostActivityChecker; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import javax.inject.Inject; -import java.util.Arrays; +import java.util.Collections; import java.util.List; public class KVMInvestigator extends AdapterBase implements Investigator { @@ -54,13 +52,15 @@ public class KVMInvestigator extends AdapterBase implements Investigator { private HAManager haManager; @Inject private DataStoreProviderManager dataStoreProviderMgr; + @Inject + private KVMHostActivityChecker hostActivityChecker; @Override public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM { if (haManager.isHAEligible(host)) { return haManager.isVMAliveOnHost(host); } - Status status = isAgentAlive(host); + Status status = getHostAgentStatus(host); logger.debug("HA: HOST is ineligible legacy state {} for host {}", status, host); if (status == null) { throw new UnknownVM(); @@ -73,86 +73,41 @@ public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws Unkno } @Override - public Status isAgentAlive(Host agent) { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + public Status getHostAgentStatus(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { return null; } - if (haManager.isHAEligible(agent)) { - return haManager.getHostStatus(agent); + if (haManager.isHAEligible(host)) { + return haManager.getHostStatusFromHAConfig(host); } - List clusterPools = _storagePoolDao.findPoolsInClusters(Arrays.asList(agent.getClusterId()), null); - boolean storageSupportHA = storageSupportHa(clusterPools); - if (!storageSupportHA) { - List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType()); - storageSupportHA = storageSupportHa(zonePools); + List clusterPools = _storagePoolDao.findPoolsInClusters(Collections.singletonList(host.getClusterId()), null); + boolean storageSupportsHA = storageSupportsHA(clusterPools); + if (!storageSupportsHA) { + List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), host.getHypervisorType()); + storageSupportsHA = storageSupportsHA(zonePools); } - if (!storageSupportHA) { - logger.warn("Agent investigation was requested on host {}, but host does not support investigation because it has no NFS storage. Skipping investigation.", agent); + if (!storageSupportsHA) { + logger.warn("Agent investigation was requested on host {}, but host does not support investigation" + + " because it has no HA supported storage. Skipping investigation.", host); return null; } - Status hostStatus = null; - Status neighbourStatus = null; - boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); - CheckOnHostCommand cmd = new CheckOnHostCommand(agent, reportFailureIfOneStorageIsDown); - - try { - Answer answer = _agentMgr.easySend(agent.getId(), cmd); - if (answer != null) { - hostStatus = answer.getResult() ? Status.Down : Status.Up; - } - } catch (Exception e) { - logger.debug("Failed to send command to host: {}", agent); - } - if (hostStatus == null) { - hostStatus = Status.Disconnected; - } - - List neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); - for (HostVO neighbor : neighbors) { - if (neighbor.getId() == agent.getId() - || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { - continue; - } - logger.debug("Investigating host:{} via neighbouring host:{}", agent, neighbor); - try { - Answer answer = _agentMgr.easySend(neighbor.getId(), cmd); - if (answer != null) { - neighbourStatus = answer.getResult() ? Status.Down : Status.Up; - logger.debug("Neighbouring host:{} returned status:{} for the investigated host:{}", neighbor, neighbourStatus, agent); - if (neighbourStatus == Status.Up) { - break; - } - } - } catch (Exception e) { - logger.debug("Failed to send command to host: {}", neighbor); - } - } - if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Disconnected; - } - if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Down; - } - logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, agent); - return hostStatus; + return hostActivityChecker.getHostAgentStatus(host); } - private boolean storageSupportHa(List pools) { - boolean storageSupportHA = false; + private boolean storageSupportsHA(List pools) { for (StoragePoolVO pool : pools) { DataStoreProvider storeProvider = dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName()); DataStoreDriver storeDriver = storeProvider.getDataStoreDriver(); if (storeDriver instanceof PrimaryDataStoreDriver) { PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver; if (primaryStoreDriver.isStorageSupportHA(pool.getPoolType())) { - storageSupportHA = true; - break; + return true; } } } - return storageSupportHA; + return false; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java new file mode 100644 index 000000000000..2e9852f7bc1e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/ImageServerControlSocket.java @@ -0,0 +1,123 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Communicates with the cloudstack-image-server control socket via socat. + * + * Protocol: newline-delimited JSON over a Unix domain socket. + * Actions: register, unregister, status. + */ +public class ImageServerControlSocket { + private static final Logger LOGGER = LogManager.getLogger(ImageServerControlSocket.class); + static final String CONTROL_SOCKET_PATH = "/var/run/cloudstack/image-server.sock"; + private static final Gson GSON = new GsonBuilder().create(); + + private ImageServerControlSocket() { + } + + /** + * Send a JSON message to the image server control socket and return the + * parsed response, or null on communication failure. + */ + static JsonObject sendMessage(Map message) { + String json = GSON.toJson(message); + Script script = new Script("/bin/bash", LOGGER); + script.add("-c"); + script.add(String.format("echo '%s' | socat -t5 - UNIX-CONNECT:%s", + json.replace("'", "'\\''"), CONTROL_SOCKET_PATH)); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = script.execute(parser); + if (result != null) { + LOGGER.error("Control socket communication failed: {}", result); + return null; + } + String output = parser.getLines(); + if (output == null || output.trim().isEmpty()) { + LOGGER.error("Empty response from control socket"); + return null; + } + try { + return JsonParser.parseString(output.trim()).getAsJsonObject(); + } catch (Exception e) { + LOGGER.error("Failed to parse control socket response: {}", output, e); + return null; + } + } + + /** + * Register a transfer config with the image server. + * @return true if the server accepted the registration. + */ + public static boolean registerTransfer(String transferId, Map config) { + Map msg = new HashMap<>(); + msg.put("action", "register"); + msg.put("transfer_id", transferId); + msg.put("config", config); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } + + /** + * Unregister a transfer from the image server. + * @return the number of remaining active transfers, or -1 on error. + */ + public static int unregisterTransfer(String transferId) { + Map msg = new HashMap<>(); + msg.put("action", "unregister"); + msg.put("transfer_id", transferId); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return -1; + } + if (!"ok".equals(resp.has("status") ? resp.get("status").getAsString() : null)) { + return -1; + } + return resp.has("active_transfers") ? resp.get("active_transfers").getAsInt() : -1; + } + + /** + * Check whether the image server control socket is responsive. + * @return true if the server responded with status "ok". + */ + public static boolean isReady() { + Map msg = new HashMap<>(); + msg.put("action", "status"); + JsonObject resp = sendMessage(msg); + if (resp == null) { + return false; + } + return "ok".equals(resp.has("status") ? resp.get("status").getAsString() : null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 896426addca1..f30e2c779195 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -35,11 +35,9 @@ public class KVMHABase { protected Logger logger = LogManager.getLogger(getClass()); private long _timeout = 60000; /* 1 minutes */ - protected static String s_heartBeatPath; - protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); - protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + protected long _heartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); - protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); + protected long _heartBeatUpdateRetrySleepInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); public static enum PoolType { PrimaryStorage, SecondaryStorage @@ -139,7 +137,7 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) { /* Can't find the mount point? */ /* we need to mount it under poolName */ if (poolName != null) { - Script mount = new Script("/bin/bash", 60000); + Script mount = new Script("/bin/bash", _timeout); mount.add("-c"); mount.add("mount " + mountSource + " " + destPath); String result = mount.execute(); @@ -155,7 +153,6 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) { } protected String getMountPoint(HAStoragePool storagePool) { - StoragePool pool = null; String poolName = null; try { @@ -172,7 +169,6 @@ protected String getMountPoint(HAStoragePool storagePool) { } poolName = pool.getName(); } - } catch (LibvirtException e) { logger.debug("Ignoring libvirt error.", e); } finally { @@ -235,7 +231,7 @@ protected String runScriptRetry(String cmdString, OutputInterpreter interpreter) return result; } - public Boolean checkingHeartBeat() { + public Boolean hasHeartBeat() { // TODO Auto-generated method stub return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index db6190fa8f28..0ee59c95da39 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -26,44 +26,43 @@ public class KVMHAChecker extends KVMHABase implements Callable { private List storagePools; private HostTO host; - private boolean reportFailureIfOneStorageIsDown; + private boolean reportIfHeartBeatFailedForOneStoragePool; - public KVMHAChecker(List pools, HostTO host, boolean reportFailureIfOneStorageIsDown) { + public KVMHAChecker(List pools, HostTO host, boolean reportIfHeartBeatFailedForOneStoragePool) { this.storagePools = pools; this.host = host; - this.reportFailureIfOneStorageIsDown = reportFailureIfOneStorageIsDown; + this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool; } /* - * True means heartbeaing is on going, or we can't get it's status. False - * means heartbeating is stopped definitely + * True means heart beating is on going, or we can't get it's status. + * False means heart beating is stopped definitely. */ @Override - public Boolean checkingHeartBeat() { - boolean validResult = false; - - String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); - - logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools)); + public Boolean hasHeartBeat() { + String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), + storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); + logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools); + boolean heartBeatCheckResult = false; for (HAStoragePool pool : storagePools) { - validResult = pool.getPool().checkingHeartBeat(pool, host); - if (reportFailureIfOneStorageIsDown && !validResult) { + heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host); + if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) { break; } } - if (!validResult) { - logger.warn(String.format("All checks with KVMHAChecker for %s considered it as dead. It may cause a shutdown of the host.", hostAndPools)); + if (!heartBeatCheckResult) { + logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools); } - return validResult; + return heartBeatCheckResult; } @Override public Boolean call() throws Exception { // logger.addAppender(new org.apache.log4j.ConsoleAppender(new // org.apache.log4j.PatternLayout(), "System.out")); - return checkingHeartBeat(); + return hasHeartBeat(); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index cf407bfc08a8..9f1b849e9727 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -18,7 +18,7 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; -import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.utils.script.Script; import org.libvirt.Connect; import org.libvirt.LibvirtException; @@ -34,60 +34,51 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { - private final Map storagePool = new ConcurrentHashMap<>(); + private final Map haStoragePools = new ConcurrentHashMap<>(); private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; - public KVMHAMonitor(HAStoragePool pool, String host, String scriptPath) { - if (pool != null) { - storagePool.put(pool.getPoolUUID(), pool); - } + public KVMHAMonitor(String host) { hostPrivateIp = host; - configureHeartBeatPath(scriptPath); - rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } - private static synchronized void configureHeartBeatPath(String scriptPath) { - KVMHABase.s_heartBeatPath = scriptPath; - } - public void addStoragePool(HAStoragePool pool) { - synchronized (storagePool) { - storagePool.put(pool.getPoolUUID(), pool); + synchronized (haStoragePools) { + haStoragePools.put(pool.getPoolUUID(), pool); } } public void removeStoragePool(String uuid) { - synchronized (storagePool) { - HAStoragePool pool = storagePool.get(uuid); + synchronized (haStoragePools) { + HAStoragePool pool = haStoragePools.get(uuid); if (pool != null) { Script.runSimpleBashScript("umount " + pool.getMountDestPath()); - storagePool.remove(uuid); + haStoragePools.remove(uuid); } } } public List getStoragePools() { - synchronized (storagePool) { - return new ArrayList<>(storagePool.values()); + synchronized (haStoragePools) { + return new ArrayList<>(haStoragePools.values()); } } public HAStoragePool getStoragePool(String uuid) { - synchronized (storagePool) { - return storagePool.get(uuid); + synchronized (haStoragePools) { + return haStoragePools.get(uuid); } } protected void runHeartBeat() { - synchronized (storagePool) { + synchronized (haStoragePools) { Set removedPools = new HashSet<>(); - for (String uuid : storagePool.keySet()) { - HAStoragePool primaryStoragePool = storagePool.get(uuid); - if (primaryStoragePool.getPool().getType() == StoragePoolType.NetworkFilesystem) { - checkForNotExistingPools(removedPools, uuid); + for (String uuid : haStoragePools.keySet()) { + HAStoragePool primaryStoragePool = haStoragePools.get(uuid); + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(primaryStoragePool.getPool().getType())) { + checkForNotExistingLibvirtStoragePools(removedPools, uuid); if (removedPools.contains(uuid)) { continue; } @@ -96,7 +87,7 @@ protected void runHeartBeat() { result = executePoolHeartBeatCommand(uuid, primaryStoragePool, result); if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + logger.warn("Write heartbeat for pool [{}] failed: {}; stopping cloudstack-agent.", uuid, result); primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false);; } } @@ -109,45 +100,43 @@ protected void runHeartBeat() { } private String executePoolHeartBeatCommand(String uuid, HAStoragePool primaryStoragePool, String result) { - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + for (int attempt = 1; attempt <= _heartBeatUpdateMaxTries; attempt++) { result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - - if (result != null) { - logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { + if (result == null) { break; } + logger.warn("Write heartbeat for pool [{}] failed: {}; try: {} of {}.", uuid, result, attempt, _heartBeatUpdateMaxTries); + try { + Thread.sleep(_heartBeatUpdateRetrySleepInMs); + } catch (InterruptedException e) { + logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } } return result; } - private void checkForNotExistingPools(Set removedPools, String uuid) { + private void checkForNotExistingLibvirtStoragePools(Set removedPools, String uuid) { try { Connect conn = LibvirtConnection.getConnection(); StoragePool storage = conn.storagePoolLookupByUUIDString(uuid); if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { if (storage == null) { - logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + logger.debug("Libvirt storage pool [{}] not found, removing from HA list.", uuid); } else { - logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + logger.debug("Libvirt storage pool [{}] found, but not running, removing from HA list.", uuid); } removedPools.add(uuid); } - logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { - logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); if (e.toString().contains("pool not found")) { - logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + logger.debug("Removing pool [{}] from HA monitor since it was deleted.", uuid); removedPools.add(uuid); } } @@ -160,11 +149,10 @@ public void run() { runHeartBeat(); try { - Thread.sleep(_heartBeatUpdateFreq); + Thread.sleep(_heartBeatUpdateFreqInMs); } catch (InterruptedException e) { logger.debug("[IGNORED] Interrupted between heartbeats.", e); } } } - } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index e6937b515e95..c13be64a3a3e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -39,12 +39,12 @@ public KVMHAVMActivityChecker(final HAStoragePool pool, final HostTO host, final } @Override - public Boolean checkingHeartBeat() { - return this.storagePool.getPool().vmActivityCheck(storagePool, host, activityScriptTimeout, volumeUuidList, vmActivityCheckPath, suspectTimeInSeconds); + public Boolean hasHeartBeat() { + return this.storagePool.getPool().hasVmActivity(storagePool, host, activityScriptTimeout, volumeUuidList, vmActivityCheckPath, suspectTimeInSeconds); } @Override public Boolean call() throws Exception { - return checkingHeartBeat(); + return hasHeartBeat(); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 46cf1da461e7..d0a162e0f6ff 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -18,6 +18,9 @@ import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_VDDK_LIB_DIR; +import static com.cloud.host.Host.HOST_VDDK_SUPPORT; +import static com.cloud.host.Host.HOST_VDDK_VERSION; import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import static org.apache.cloudstack.utils.linux.KVMHostInfo.isHostS390x; @@ -121,6 +124,7 @@ import org.libvirt.LibvirtException; import org.libvirt.MemoryStatistic; import org.libvirt.Network; +import org.libvirt.SchedLongParameter; import org.libvirt.SchedParameter; import org.libvirt.SchedUlongParameter; import org.libvirt.Secret; @@ -365,6 +369,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win"; public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; + public static final String VDDK_AUTODETECT_PATH_CMD = "find / -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; public static final int LIBVIRT_CGROUP_CPU_SHARES_MIN = 2; public static final int LIBVIRT_CGROUP_CPU_SHARES_MAX = 262144; @@ -382,6 +387,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s --metadata"; + public static final int IMAGE_SERVER_DEFAULT_PORT = 54322; + public static final String IMAGE_SERVER_SYSTEMD_UNIT_NAME = "cloudstack-image-server"; + protected int qcow2DeltaMergeTimeout; private String modifyVlanPath; @@ -395,6 +403,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String heartBeatPath; private String vmActivityCheckPath; private String nasBackupPath; + private String imageServerPath; + private boolean imageServerTlsEnabled = false; + private String imageServerListenAddress; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -809,6 +820,18 @@ public String getNasBackupPath() { return nasBackupPath; } + public String getImageServerPath() { + return imageServerPath; + } + + public boolean isImageServerTlsEnabled() { + return imageServerTlsEnabled; + } + + public String getImageServerListenAddress() { + return imageServerListenAddress; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -879,16 +902,41 @@ protected enum HealthCheckResult { SUCCESS, FAILURE, IGNORE } + public enum CpuSchedulerParameter { + CPU_SHARES("cpu_shares"), PERIOD("vcpu_period"), QUOTA("vcpu_quota"); + + private String name; + + CpuSchedulerParameter(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return getName(); + } + } + protected BridgeType bridgeType; protected StorageSubsystemCommandHandler storageHandler; private boolean convertInstanceVerboseMode = false; private Map convertInstanceEnv = null; + private String vddkLibDir = null; + private static final String libguestfsBackend = "direct"; protected boolean dpdkSupport = false; protected String dpdkOvsPath; protected String directDownloadTemporaryDownloadPath; protected String cachePath; + private String vddkTransports = null; + private String vddkThumbprint = null; + private String vddkVersion = null; + private String detectedPasswordFileOption = null; protected String javaTempDir = System.getProperty("java.io.tmpdir"); private String getEndIpFromStartIp(final String startIp, final int numIps) { @@ -953,6 +1001,26 @@ public Map getConvertInstanceEnv() { return convertInstanceEnv; } + public String getVddkLibDir() { + return vddkLibDir; + } + + public String getLibguestfsBackend() { + return libguestfsBackend; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public String getVddkVersion() { + return vddkVersion; + } + /** * Defines resource's public and private network interface according to what is configured in agent.properties. */ @@ -1027,6 +1095,9 @@ public boolean configure(final String name, final Map params) th cachePath = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_CACHE_LOCATION); + imageServerTlsEnabled = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_TLS_ENABLED); + imageServerListenAddress = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.IMAGE_SERVER_LISTEN_ADDRESS); + params.put("domr.scripts.dir", domrScriptsDir); virtRouterResource = new VirtualRoutingResource(this); @@ -1065,11 +1136,6 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find patch.sh"); } - heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (heartBeatPath == null) { - throw new ConfigurationException("Unable to find kvmheartbeat.sh"); - } - createVmPath = Script.findScript(storageScriptsDir, "createvm.sh"); if (createVmPath == null) { throw new ConfigurationException("Unable to find the createvm.sh"); @@ -1095,6 +1161,12 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find nasbackup.sh"); } + String imageServerMain = Script.findScript(kvmScriptsDir, "imageserver/__main__.py"); + if (imageServerMain == null) { + throw new ConfigurationException("Unable to find imageserver package"); + } + imageServerPath = new File(imageServerMain).getParent(); + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); @@ -1158,6 +1230,37 @@ public boolean configure(final String name, final Map params) th setConvertInstanceEnv(convertEnvTmpDir, convertEnvVirtv2vTmpDir); + vddkLibDir = StringUtils.trimToNull(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_LIB_DIR)); + if (StringUtils.isNotBlank(vddkLibDir) && !isVddkLibDirValid(vddkLibDir)) { + LOGGER.warn("Configured VDDK library dir [{}] is invalid (missing lib64/libvixDiskLib.so), attempting auto-detection", vddkLibDir); + vddkLibDir = null; + } + if (StringUtils.isBlank(vddkLibDir)) { + vddkLibDir = detectVddkLibDir(); + } + if (StringUtils.isNotBlank(vddkLibDir)) { + LOGGER.info("Detected VDDK library dir: {}", vddkLibDir); + } else { + LOGGER.warn("Could not detect a valid VDDK library dir; VDDK conversion will be unavailable"); + } + + vddkVersion = detectVddkVersion(); + if (StringUtils.isNotBlank(vddkVersion)) { + LOGGER.info("Detected nbdkit VDDK plugin version: {}", vddkVersion); + } + + vddkTransports = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_TRANSPORTS)); + vddkThumbprint = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_THUMBPRINT)); + + detectedPasswordFileOption = detectPasswordFileOption(); + if (StringUtils.isNotBlank(detectedPasswordFileOption)) { + LOGGER.info("Detected virt-v2v password option: {}", detectedPasswordFileOption); + } else { + LOGGER.warn("Could not detect virt-v2v password option, VDDK conversions may fail"); + } + pool = (String)params.get("pool"); if (pool == null) { pool = "/root"; @@ -1332,9 +1435,9 @@ public boolean configure(final String name, final Map params) th final String[] info = NetUtils.getNetworkParams(privateNic); - kvmhaMonitor = new KVMHAMonitor(null, info[0], heartBeatPath); - final Thread ha = new Thread(kvmhaMonitor); - ha.start(); + kvmhaMonitor = new KVMHAMonitor(info[0]); + final Thread haMonitorThread = new Thread(kvmhaMonitor); + haMonitorThread.start(); storagePoolManager = new KVMStoragePoolManager(storageLayer, kvmhaMonitor); @@ -2909,23 +3012,61 @@ protected String getUuid(String uuid) { protected void setQuotaAndPeriod(VirtualMachineTO vmTO, CpuTuneDef ctd) { if (vmTO.isLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) { Double cpuQuotaPercentage = vmTO.getCpuQuotaPercentage(); - int period = CpuTuneDef.DEFAULT_PERIOD; - int quota = (int) (period * cpuQuotaPercentage); - if (quota < CpuTuneDef.MIN_QUOTA) { - LOGGER.info("Calculated quota (" + quota + ") below the minimum (" + CpuTuneDef.MIN_QUOTA + ") for VM domain " + vmTO.getUuid() + ", setting it to minimum " + - "and calculating period instead of using the default"); - quota = CpuTuneDef.MIN_QUOTA; - period = (int) ((double) quota / cpuQuotaPercentage); - if (period > CpuTuneDef.MAX_PERIOD) { - LOGGER.info("Calculated period (" + period + ") exceeds the maximum (" + CpuTuneDef.MAX_PERIOD + - "), setting it to the maximum"); - period = CpuTuneDef.MAX_PERIOD; - } + Pair periodAndQuota = getPeriodAndQuota(cpuQuotaPercentage); + ctd.setPeriod(periodAndQuota.first()); + ctd.setQuota(periodAndQuota.second()); + LOGGER.info("Setting quota = [{}] and period = [{}] to VM domain [{}].", periodAndQuota.second(), periodAndQuota.first(), vmTO.getUuid()); + } + } + + /** + * Calculates the CPU period and quota based on the quota percentage defined by the Management Server + * @param cpuQuotaPercentage CPU quota percentage defined by the Management Server + * @return The period and quota to be defined for the VM's domain + */ + protected Pair getPeriodAndQuota(double cpuQuotaPercentage) { + int period = CpuTuneDef.DEFAULT_PERIOD; + long quota = (long) (period * cpuQuotaPercentage); + if (quota < CpuTuneDef.MIN_QUOTA) { + LOGGER.info("Calculated quota ({}) below the minimum ({}), setting it to minimum and calculating period instead of using the default", quota, CpuTuneDef.MIN_QUOTA); + quota = CpuTuneDef.MIN_QUOTA; + period = (int) ((double) quota / cpuQuotaPercentage); + if (period > CpuTuneDef.MAX_PERIOD) { + LOGGER.info("Calculated period ({}) exceeds the maximum ({}), setting it to the maximum", period, CpuTuneDef.MAX_PERIOD); + period = CpuTuneDef.MAX_PERIOD; } - ctd.setQuota(quota); - ctd.setPeriod(period); - LOGGER.info("Setting quota=" + quota + ", period=" + period + " to VM domain " + vmTO.getUuid()); } + + LOGGER.info("Calculated period = [{}] and quota = [{}] given the [{}] quota percentage.", period, quota, cpuQuotaPercentage); + return new Pair<>(period, quota); + } + + /** + * Dynamically updates the domain's "vcpu_quota" and "period" fields of the CPU tune definition. + * This is required because the values of the fields must change according to the new CPU speed of the VM. + * When the CPU limitation is removed from the domain, the "vcpu_quota" field is set to 17,592,186,044,415. + * @param domain VM's domain. + * @param vmTO VM's transfer object, which contains the required fields to update the "vcpu_quota" and "period" fields. + * @param limitCpuUseChange Indicates whether the CPU limitation for the VM has changed. + * @throws org.libvirt.LibvirtException + **/ + public void updateCpuQuotaAndPeriod(Domain domain, VirtualMachineTO vmTO, boolean limitCpuUseChange) throws LibvirtException { + if (hypervisorLibvirtVersion < MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE || (!limitCpuUseChange && !vmTO.isLimitCpuUse())) { + logger.info("Not updating the [{}] and [{}] for the [{}] domain, because [{}].", + CpuSchedulerParameter.QUOTA, CpuSchedulerParameter.PERIOD, domain.getName(), hypervisorLibvirtVersion < MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE ? + "the current Libvirt version does not support CPU tune" : "it was not requested to remove, change or apply CPU limitation for the instance."); + return; + } + + if (limitCpuUseChange && !vmTO.isLimitCpuUse()) { + logger.info("Updating the [{}] of the [{}] domain to [{}], because CPU limitation has been removed.", CpuSchedulerParameter.QUOTA, domain.getName(), CpuTuneDef.MAX_CPU_QUOTA); + LibvirtComputingResource.setQuota(domain, CpuTuneDef.MAX_CPU_QUOTA); + return; + } + + Pair periodAndQuota = getPeriodAndQuota(vmTO.getCpuQuotaPercentage()); + LibvirtComputingResource.setPeriod(domain, periodAndQuota.first()); + LibvirtComputingResource.setQuota(domain, periodAndQuota.second()); } protected void enlightenWindowsVm(VirtualMachineTO vmTO, FeaturesDef features) { @@ -3389,10 +3530,10 @@ protected GuestResourceDef createGuestResourceDef(VirtualMachineTO vmTO){ grd.setMemBalloning(!noMemBalloon); - Long maxRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMaxRam()); - - grd.setMemorySize(maxRam); - grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, maxRam)); + long requestedRam = ByteScaleUtils.bytesToKibibytes(vmTO.getRequestedRam()); + long minRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMinRam()); + grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, requestedRam, minRam)); + grd.setMaxMemory(ByteScaleUtils.bytesToKibibytes(vmTO.getMaxRam())); int vcpus = vmTO.getCpus(); Integer maxVcpus = vmTO.getVcpuMaxLimit(); @@ -3403,18 +3544,19 @@ protected GuestResourceDef createGuestResourceDef(VirtualMachineTO vmTO){ return grd; } - protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long maxRam) { - long retVal = maxRam; + protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long requestedRam, long minRam) { if (noMemBalloon) { - LOGGER.warn(String.format("Setting VM's [%s] current memory as max memory [%s] due to memory ballooning is disabled. If you are using a custom service offering, verify if memory ballooning really should be disabled.", vmTO.toString(), maxRam)); - } else if (vmTO != null && vmTO.getType() != VirtualMachine.Type.User) { - LOGGER.warn(String.format("Setting System VM's [%s] current memory as max memory [%s].", vmTO.toString(), maxRam)); - } else { - long minRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMinRam()); - LOGGER.debug(String.format("Setting VM's [%s] current memory as min memory [%s] due to memory ballooning is enabled.", vmTO.toString(), minRam)); - retVal = minRam; + LOGGER.warn("Setting VM's [{}] current memory as requested memory [{}] due to memory ballooning is disabled.", vmTO.toString(), requestedRam); + return requestedRam; + } + + if (vmTO != null && vmTO.getType() != VirtualMachine.Type.User) { + LOGGER.warn("Setting System VM's [{}] current memory as requested memory [{}].", vmTO.toString(), requestedRam); + return requestedRam; } - return retVal; + + LOGGER.debug("Setting VM's [{}] current memory as min memory [{}] due to memory ballooning is enabled.", vmTO.toString(), minRam); + return minRam; } /** @@ -4229,6 +4371,13 @@ public StartupCommand[] initialize() { cmd.setHostTags(getHostTags()); boolean instanceConversionSupported = hostSupportsInstanceConversion(); cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + cmd.getHostDetails().put(HOST_VDDK_SUPPORT, String.valueOf(hostSupportsVddk())); + if (StringUtils.isNotBlank(vddkLibDir)) { + cmd.getHostDetails().put(HOST_VDDK_LIB_DIR, vddkLibDir); + } + if (StringUtils.isNotBlank(vddkVersion)) { + cmd.getHostDetails().put(HOST_VDDK_VERSION, vddkVersion); + } if (instanceConversionSupported) { cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); } @@ -5227,6 +5376,24 @@ public void removeCheckpointsOnVm(String vmName, String volumeUuid, List logger.debug("Removed all checkpoints of volume [{}] on VM [{}].", volumeUuid, vmName); } + public Map getDiskPathLabelMap(String vmName) { + try { + Connect conn = LibvirtConnection.getConnectionByVmName(vmName); + List disks = getDisks(conn, vmName); + Map diskPathLabelMap = new HashMap<>(); + for (DiskDef disk : disks) { + if (disk.getDeviceType() != DeviceType.DISK) { + continue; + } + diskPathLabelMap.put(disk.getDiskPath(), disk.getDiskLabel()); + } + return diskPathLabelMap; + } catch (LibvirtException e) { + logger.error("Failed to get disk path label map for VM [{}] due to: [{}].", vmName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + public boolean recreateCheckpointsOnVm(List volumes, String vmName, Connect conn) { logger.debug("Trying to recreate checkpoints on VM [{}] with volumes [{}].", vmName, volumes); try { @@ -5950,6 +6117,66 @@ public boolean hostSupportsInstanceConversion() { return exitValue == 0; } + public boolean hostSupportsVddk() { + return hostSupportsVddk(null); + } + + public boolean hostSupportsVddk(String overriddenVddkLibDir) { + String effectiveVddkLibDir = StringUtils.trimToNull(overriddenVddkLibDir); + if (StringUtils.isBlank(effectiveVddkLibDir)) { + effectiveVddkLibDir = StringUtils.trimToNull(vddkLibDir); + } + if (StringUtils.isBlank(effectiveVddkLibDir) || !isVddkLibDirValid(effectiveVddkLibDir)) { + effectiveVddkLibDir = detectVddkLibDir(); + } + return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(detectVddkVersion()); + } + + protected boolean isVddkLibDirValid(String path) { + if (StringUtils.isBlank(path)) { + return false; + } + File libDir = new File(path, "lib64"); + if (!libDir.isDirectory()) { + return false; + } + File[] libs = libDir.listFiles((dir, name) -> name.startsWith("libvixDiskLib.so")); + return libs != null && libs.length > 0; + } + + protected String detectVddkLibDir() { + String detectedPath = StringUtils.trimToNull(Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD)); + if (StringUtils.isNotBlank(detectedPath) && isVddkLibDirValid(detectedPath)) { + return detectedPath; + } + return null; + } + + protected String detectVddkVersion() { + try { + ProcessBuilder pb = new ProcessBuilder("nbdkit", "vddk", "--version"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (StringUtils.isBlank(output)) { + return null; + } + + for (String line : output.split("\\R")) { + String trimmed = StringUtils.trimToEmpty(line); + if (trimmed.startsWith("vddk ")) { + return StringUtils.trimToNull(trimmed.substring("vddk ".length())); + } + } + return null; + } catch (Exception e) { + LOGGER.error("Failed to detect vddk version: {}", e.getMessage()); + return null; + } + } + public boolean hostSupportsWindowsGuestConversion() { if (isUbuntuOrDebianHost()) { int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); @@ -5964,6 +6191,40 @@ public boolean hostSupportsOvfExport() { return exitValue == 0; } + /** + * Detect which password option virt-v2v supports by examining its --help output + * @return "-ip" if supported (virt-v2v >= 2.8.1), "--password-file" if older version, or null if detection fails + */ + protected String detectPasswordFileOption() { + try { + ProcessBuilder pb = new ProcessBuilder("virt-v2v", "--help"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (output.contains("-ip ")) { + return "-ip"; + } else if (output.contains("--password-file")) { + return "--password-file"; + } else { + LOGGER.error("virt-v2v does not support -ip or --password-file"); + return null; + } + } catch (Exception e) { + LOGGER.error("Failed to detect virt-v2v password option: {}", e.getMessage()); + return null; + } + } + + /** + * Get the detected password file option for virt-v2v + * @return the password option ("-ip" or "--password-file") or null if not detected + */ + public String getDetectedPasswordFileOption() { + return detectedPasswordFileOption; + } + public String getHostVirtV2vVersion() { if (!hostSupportsInstanceConversion()) { return ""; @@ -6098,29 +6359,59 @@ public static long countDomainRunningVcpus(Domain dm) throws LibvirtException { **/ public static Integer getCpuShares(Domain dm) throws LibvirtException { for (SchedParameter c : dm.getSchedulerParameters()) { - if (c.field.equals("cpu_shares")) { + if (c.field.equals(CpuSchedulerParameter.CPU_SHARES.getName())) { return Integer.parseInt(c.getValueAsString()); } } - LOGGER.warn(String.format("Could not get cpu_shares of domain: [%s]. Returning default value of 0. ", dm.getName())); + LOGGER.warn("Could not get [{}] of domain: [{}]. Returning default value of 0. ", CpuSchedulerParameter.CPU_SHARES.getName(), dm.getName()); return 0; } /** - * Sets the cpu_shares (priority) of the running VM
+ * Updates the cpu_shares (priority) of the running VM. * @param dm domain of the VM. * @param cpuShares new priority of the running VM. - * @throws org.libvirt.LibvirtException **/ public static void setCpuShares(Domain dm, Integer cpuShares) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.CPU_SHARES.getName(), dm.getName(), cpuShares); SchedUlongParameter[] params = new SchedUlongParameter[1]; params[0] = new SchedUlongParameter(); - params[0].field = "cpu_shares"; + params[0].field = CpuSchedulerParameter.CPU_SHARES.getName(); params[0].value = cpuShares; dm.setSchedulerParameters(params); } + /** + * Updates the period of the running VM. + * @param domain domain of the VM. + * @param period new period of the running VM. + **/ + public static void setPeriod(Domain domain, int period) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.PERIOD.getName(), domain.getName(), period); + SchedUlongParameter[] params = new SchedUlongParameter[1]; + params[0] = new SchedUlongParameter(); + params[0].field = CpuSchedulerParameter.PERIOD.getName(); + params[0].value = period; + + domain.setSchedulerParameters(params); + } + + /** + * Updates the quota of the running VM. + * @param domain domain of the VM. + * @param quota new quota of the running VM. + **/ + public static void setQuota(Domain domain, long quota) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.QUOTA.getName(), domain.getName(), quota); + SchedLongParameter[] params = new SchedLongParameter[1]; + params[0] = new SchedLongParameter(); + params[0].field = CpuSchedulerParameter.QUOTA.getName(); + params[0].value = quota; + + domain.setSchedulerParameters(params); + } + /** * Set up a libvirt secret for a volume. If Libvirt says that a secret already exists for this volume path, we use its uuid. * The UUID of the secret needs to be prescriptive such that we can register the same UUID on target host during live migration diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index 4d823783a99a..e114669b8b56 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -542,7 +542,7 @@ private void extractCpuTuneDef(final Element rootElement) { final String quota = getTagValue("quota", cpuTuneDefElement); if (StringUtils.isNotBlank(quota)) { - cpuTuneDef.setQuota((Integer.parseInt(quota))); + cpuTuneDef.setQuota((Long.parseLong(quota))); } final String period = getTagValue("period", cpuTuneDefElement); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java index 08086859fb70..d3765c01ccbc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java @@ -79,28 +79,47 @@ private void generatePciXml(StringBuilder gpuBuilder) { gpuBuilder.append(" \n"); gpuBuilder.append(" \n"); - // Parse the bus address (e.g., 00:02.0) into domain, bus, slot, function - String domain = "0x0000"; - String bus = "0x00"; - String slot = "0x00"; - String function = "0x0"; + // Parse the bus address into domain, bus, slot, function. Two input formats are accepted: + // - "dddd:bb:ss.f" full PCI address with domain (e.g. 0000:00:02.0) + // - "bb:ss.f" legacy short BDF; domain defaults to 0000 + // Each segment is parsed as a hex integer and formatted with fixed widths + // (domain: 4 hex digits, bus/slot: 2 hex digits, function: 1 hex digit) to + // produce canonical libvirt XML values regardless of input casing or padding. + int domainVal = 0, busVal = 0, slotVal = 0, funcVal = 0; if (busAddress != null && !busAddress.isEmpty()) { String[] parts = busAddress.split(":"); - if (parts.length > 1) { - bus = "0x" + parts[0]; - String[] slotFunctionParts = parts[1].split("\\."); - if (slotFunctionParts.length > 0) { - slot = "0x" + slotFunctionParts[0]; - if (slotFunctionParts.length > 1) { - function = "0x" + slotFunctionParts[1].trim(); - } + try { + String slotFunction; + if (parts.length == 3) { + domainVal = Integer.parseInt(parts[0], 16); + busVal = Integer.parseInt(parts[1], 16); + slotFunction = parts[2]; + } else if (parts.length == 2) { + busVal = Integer.parseInt(parts[0], 16); + slotFunction = parts[1]; + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); } + String[] sf = slotFunction.split("\\."); + if (sf.length == 2) { + slotVal = Integer.parseInt(sf[0], 16); + funcVal = Integer.parseInt(sf[1].trim(), 16); + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'", e); } } + String domain = String.format("0x%04x", domainVal); + String bus = String.format("0x%02x", busVal); + String slot = String.format("0x%02x", slotVal); + String function = String.format("0x%x", funcVal); + gpuBuilder.append("

\n"); + .append(slot).append("' function='").append(function).append("'/>\n"); gpuBuilder.append(" \n"); gpuBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 097a9b8dd322..bf8b1af6c18d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -443,15 +443,15 @@ public String toString() { } public static class GuestResourceDef { - private long memory; + private long maxMemory; private long currentMemory = -1; private int vcpu = -1; private int maxVcpu = -1; private boolean memoryBalloning = false; private int memoryBalloonStatsPeriod = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_MEMBALLOON_STATS_PERIOD); - public void setMemorySize(long mem) { - this.memory = mem; + public void setMaxMemory(long mem) { + this.maxMemory = mem; } public void setCurrentMem(long currMem) { @@ -484,8 +484,8 @@ public String toString() { response.append(String.format("%s\n", this.currentMemory)); response.append(String.format("%s\n", this.currentMemory)); - if (this.memory > this.currentMemory) { - response.append(String.format("%s\n", this.memory)); + if (this.maxMemory > this.currentMemory) { + response.append(String.format("%s\n", this.maxMemory)); response.append(String.format(" \n", this.maxVcpu - 1, this.currentMemory)); } @@ -1920,11 +1920,12 @@ public String toString() { public static class CpuTuneDef { private int _shares = 0; - private int quota = 0; + private long quota = 0; private int period = 0; static final int DEFAULT_PERIOD = 10000; static final int MIN_QUOTA = 1000; static final int MAX_PERIOD = 1000000; + public static final long MAX_CPU_QUOTA = 17592186044415L; public void setShares(int shares) { _shares = shares; @@ -1934,11 +1935,11 @@ public int getShares() { return _shares; } - public int getQuota() { + public long getQuota() { return quota; } - public void setQuota(int quota) { + public void setQuota(long quota) { this.quota = quota; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java index b94b48300036..de9341715f02 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -30,7 +30,15 @@ public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper pools = monitor.getStoragePools(); final HostTO host = command.getHost(); - final KVMHAChecker ha = new KVMHAChecker(pools, host, command.isCheckFailedOnOneStorage()); + + final KVMHAChecker ha = new KVMHAChecker(pools, host, command.shouldReportIfHeartBeatFailedForOneStoragePool()); final Future future = executors.submit(ha); try { - final Boolean result = future.get(); - if (result) { - return new Answer(command, false, "Heart is beating..."); + final Boolean hasHeartBeat = future.get(); + if (hasHeartBeat) { + return new CheckOnHostAnswer(command, true, "Heart is beating"); } else { - return new Answer(command); + return new CheckOnHostAnswer(command, "Heart is not beating"); } } catch (final InterruptedException e) { - return new Answer(command, false, "CheckOnHostCommand: can't get status of host: InterruptedException"); + return new CheckOnHostAnswer(command, "CheckOnHostCommand: can't get status of host: InterruptedException"); } catch (final ExecutionException e) { - return new Answer(command, false, "CheckOnHostCommand: can't get status of host: ExecutionException"); + return new CheckOnHostAnswer(command, "CheckOnHostCommand: can't get status of host: ExecutionException"); } } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index a708d441be59..b132f6d98ce9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -48,9 +48,9 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid()); - if (primaryPool.isPoolSupportHA()){ - final HAStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + if (primaryPool.isPoolSupportHA()) { + final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid()); + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java index 99005755cbbe..aa6a37b6561d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java @@ -45,11 +45,10 @@ public Answer execute(final CheckVirtualMachineCommand command, final LibvirtCom Integer vncPort = null; if (state == PowerState.PowerOn) { vncPort = libvirtComputingResource.getVncPort(conn, command.getVmName()); - } - - Domain vm = conn.domainLookupByName(command.getVmName()); - if (state == PowerState.PowerOn && DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { - return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + Domain vm = conn.domainLookupByName(command.getVmName()); + if (DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { + return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + } } return new CheckVirtualMachineAnswer(command, state, vncPort); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 55225ba086c3..6788516df741 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -47,7 +47,10 @@ @ResourceWrapper(handles = CheckVolumeCommand.class) public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper { - private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem); + private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( + Storage.StoragePoolType.Filesystem, + Storage.StoragePoolType.NetworkFilesystem, + Storage.StoragePoolType.SharedMountPoint); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index 66a5f5dd7d20..f28b5eda4a63 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -20,10 +20,17 @@ import java.net.URLEncoder; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Locale; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.commons.collections4.MapUtils; @@ -51,6 +58,7 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper supportedInstanceConvertSourceHypervisors = List.of(Hypervisor.HypervisorType.VMware); + private static final Pattern SHA1_FINGERPRINT_PATTERN = Pattern.compile("(?i)(?:SHA1\\s+)?Fingerprint\\s*=\\s*([0-9A-F:]+)"); @Override public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { @@ -61,7 +69,8 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); long timeout = (long) cmd.getWait() * 1000; String extraParams = cmd.getExtraParams(); - String originalVMName = cmd.getOriginalVMName(); // For logging purposes, as the sourceInstance may have been cloned + boolean useVddk = cmd.isUseVddk(); + String originalVMName = cmd.getOriginalVMName(); if (cmd.getCheckConversionSupport() && !serverResource.hostSupportsInstanceConversion()) { String msg = String.format("Cannot convert the instance %s from VMware as the virt-v2v binary is not found. " + @@ -84,61 +93,75 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve logger.info(String.format("(%s) Attempting to convert the instance %s from %s to KVM", originalVMName, sourceInstanceName, sourceHypervisorType)); final String temporaryConvertPath = temporaryStoragePool.getLocalPath(); + final String temporaryConvertUuid = UUID.randomUUID().toString(); + boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); - String ovfTemplateDirOnConversionLocation; - String sourceOVFDirPath; + boolean cleanupSecondaryStorage = false; boolean ovfExported = false; - if (cmd.getExportOvfToConversionLocation()) { - String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance, originalVMName); - if (StringUtils.isBlank(exportInstanceOVAUrl)) { - String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName); - logger.error(String.format("(%s) %s", originalVMName, err)); - return new Answer(cmd, false, err); - } + String ovfTemplateDirOnConversionLocation = null; - int noOfThreads = cmd.getThreadsCountToExportOvf(); - if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { - noOfThreads = 0; - } - ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); - temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); - if (!ovfExported) { - String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); - logger.error(String.format("(%s) %s", originalVMName, err)); - return new Answer(cmd, false, err); - } - sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); - } else { - ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - } + try { + boolean result; + if (useVddk) { + logger.info("({}) Using VDDK-based conversion (direct from VMware)", originalVMName); + String vddkLibDir = resolveVddkSetting(cmd.getVddkLibDir(), serverResource.getVddkLibDir()); + if (StringUtils.isBlank(vddkLibDir)) { + String err = String.format("VDDK lib dir is not configured on the host. " + + "Set '%s' in agent.properties or in details parameter of the import api call to use VDDK-based conversion.", "vddk.lib.dir"); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + String vddkTransports = resolveVddkSetting(cmd.getVddkTransports(), serverResource.getVddkTransports()); + String configuredVddkThumbprint = resolveVddkSetting(cmd.getVddkThumbprint(), serverResource.getVddkThumbprint()); + String passwordOption = serverResource.getDetectedPasswordFileOption(); + result = performInstanceConversionUsingVddk(sourceInstance, originalVMName, temporaryConvertPath, + vddkLibDir, serverResource.getLibguestfsBackend(), vddkTransports, configuredVddkThumbprint, + timeout, verboseModeEnabled, extraParams, temporaryConvertUuid, passwordOption); + } else { + logger.info("({}) Using OVF-based conversion (export + local convert)", originalVMName); + String sourceOVFDirPath; + if (cmd.getExportOvfToConversionLocation()) { + String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance, originalVMName); - logger.info(String.format("(%s) Attempting to convert the OVF %s of the instance %s from %s to KVM", - originalVMName, ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType)); + if (StringUtils.isBlank(exportInstanceOVAUrl)) { + String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } - final String temporaryConvertUuid = UUID.randomUUID().toString(); - boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); + int noOfThreads = cmd.getThreadsCountToExportOvf(); + if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { + noOfThreads = 0; + } + ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); + temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); + + if (!ovfExported) { + String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); + } else { + ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + } + + result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, + timeout, verboseModeEnabled, extraParams, serverResource); + } - boolean cleanupSecondaryStorage = false; - try { - boolean result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, - timeout, verboseModeEnabled, extraParams, serverResource); if (!result) { - String err = String.format( - "The virt-v2v conversion for the OVF %s failed. Please check the agent logs " + - "for the virt-v2v output. Please try on a different kvm host which " + - "has a different virt-v2v version.", - ovfTemplateDirOnConversionLocation); - logger.error(String.format("(%s) %s", originalVMName, err)); + String err = String.format("Instance conversion failed for VM %s. Please check virt-v2v logs.", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); return new Answer(cmd, false, err); } return new ConvertInstanceAnswer(cmd, temporaryConvertUuid); } catch (Exception e) { - String error = String.format("Error converting instance %s from %s, due to: %s", - sourceInstanceName, sourceHypervisorType, e.getMessage()); - logger.error(String.format("(%s) %s", originalVMName, error), e); + String error = String.format("Error converting instance %s from %s, due to: %s", sourceInstanceName, sourceHypervisorType, e.getMessage()); + logger.error("({}) {}", originalVMName, error, e); cleanupSecondaryStorage = true; return new Answer(cmd, false, error); } finally { @@ -275,4 +298,198 @@ protected void addExtraParamsToScript(String extraParams, Script script) { protected String encodeUsername(String username) { return URLEncoder.encode(username, Charset.defaultCharset()); } + + private String resolveVddkSetting(String commandValue, String agentValue) { + return StringUtils.defaultIfBlank(StringUtils.trimToNull(commandValue), StringUtils.trimToNull(agentValue)); + } + + protected boolean performInstanceConversionUsingVddk(RemoteInstanceTO vmwareInstance, String originalVMName, + String temporaryConvertFolder, String vddkLibDir, + String libguestfsBackend, String vddkTransports, + String configuredVddkThumbprint, + long timeout, boolean verboseModeEnabled, String extraParams, + String temporaryConvertUuid, String passwordOption) { + + String vcenterPassword = vmwareInstance.getVcenterPassword(); + if (StringUtils.isBlank(vcenterPassword)) { + logger.error("({}) Could not determine vCenter password for {}", originalVMName, vmwareInstance.getVcenterHost()); + return false; + } + + String passwordFilePath = String.format("/tmp/v2v.pass.cloud.%s.%s", + StringUtils.defaultIfBlank(vmwareInstance.getVcenterHost(), "unknown"), + UUID.randomUUID()); + try { + Files.writeString(Path.of(passwordFilePath), vcenterPassword); + Files.setPosixFilePermissions(Path.of(passwordFilePath), Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); + logger.debug("({}) Written vCenter password to {}", originalVMName, passwordFilePath); + } catch (Exception e) { + logger.error("({}) Failed to write vCenter password file {}: {}", originalVMName, passwordFilePath, e.getMessage()); + return false; + } + + try { + String vpxUrl = buildVpxUrl(vmwareInstance); + + StringBuilder cmd = new StringBuilder(); + + cmd.append("export LIBGUESTFS_BACKEND=").append(libguestfsBackend).append(" && "); + + cmd.append("virt-v2v "); + cmd.append("--root first "); + cmd.append("-ic '").append(vpxUrl).append("' "); + if (StringUtils.isBlank(passwordOption)) { + logger.error("({}) Could not determine supported password file option for virt-v2v", originalVMName); + return false; + } + + cmd.append(passwordOption).append(" ").append(passwordFilePath).append(" "); + cmd.append("-it vddk "); + cmd.append("-io vddk-libdir=").append(vddkLibDir).append(" "); + String vddkThumbprint = StringUtils.trimToNull(configuredVddkThumbprint); + if (StringUtils.isBlank(vddkThumbprint)) { + vddkThumbprint = getVcenterThumbprint(vmwareInstance.getVcenterHost(), timeout, originalVMName); + } + if (StringUtils.isBlank(vddkThumbprint)) { + logger.error("({}) Could not determine vCenter thumbprint for {}", originalVMName, vmwareInstance.getVcenterHost()); + return false; + } + cmd.append("-io vddk-thumbprint=").append(vddkThumbprint).append(" "); + if (StringUtils.isNotBlank(vddkTransports)) { + cmd.append("-io vddk-transports=").append(vddkTransports).append(" "); + } + cmd.append(vmwareInstance.getInstanceName()).append(" "); + cmd.append("-o local "); + cmd.append("-os ").append(temporaryConvertFolder).append(" "); + cmd.append("-of qcow2 "); + cmd.append("-on ").append(temporaryConvertUuid).append(" "); + + if (verboseModeEnabled) { + cmd.append("-v "); + } + + if (StringUtils.isNotBlank(extraParams)) { + cmd.append(extraParams).append(" "); + } + + Script script = new Script("/bin/bash", timeout, logger); + script.add("-c"); + script.add(cmd.toString()); + + String logPrefix = String.format("(%s) virt-v2v vddk import", originalVMName); + OutputInterpreter.LineByLineOutputLogger outputLogger = + new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix); + + logger.info("({}) Starting virt-v2v VDDK conversion", originalVMName); + script.execute(outputLogger); + + int exitValue = script.getExitValue(); + if (exitValue != 0) { + logger.error("({}) virt-v2v failed with exit code {}", originalVMName, exitValue); + } + + return exitValue == 0; + } finally { + try { + Files.deleteIfExists(Path.of(passwordFilePath)); + logger.debug("({}) Deleted password file {}", originalVMName, passwordFilePath); + } catch (Exception e) { + logger.warn("({}) Failed to delete password file {}: {}", originalVMName, passwordFilePath, e.getMessage()); + } + } + } + + protected String getVcenterThumbprint(String vcenterHost, long timeout, String originalVMName) { + if (StringUtils.isBlank(vcenterHost)) { + return null; + } + + String endpoint = String.format("%s:443", vcenterHost); + String command = String.format("openssl s_client -connect '%s' /dev/null | " + + "openssl x509 -fingerprint -sha1 -noout", endpoint); + + Script script = new Script("/bin/bash", timeout, logger); + script.add("-c"); + script.add(command); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + script.execute(parser); + + String output = parser.getLines(); + if (script.getExitValue() != 0) { + logger.error("({}) Failed to fetch vCenter thumbprint for {}", originalVMName, vcenterHost); + return null; + } + + String thumbprint = extractSha1Fingerprint(output); + if (StringUtils.isBlank(thumbprint)) { + logger.error("({}) Failed to parse vCenter thumbprint from output for {}", originalVMName, vcenterHost); + return null; + } + return thumbprint; + } + + private String extractSha1Fingerprint(String output) { + String parsedOutput = StringUtils.trimToEmpty(output); + if (StringUtils.isBlank(parsedOutput)) { + return null; + } + + for (String line : parsedOutput.split("\\R")) { + String trimmedLine = StringUtils.trimToEmpty(line); + if (StringUtils.isBlank(trimmedLine)) { + continue; + } + + Matcher matcher = SHA1_FINGERPRINT_PATTERN.matcher(trimmedLine); + if (matcher.find()) { + return matcher.group(1).toUpperCase(Locale.ROOT); + } + + // Fallback for raw fingerprint-only output. + if (trimmedLine.matches("(?i)[0-9a-f]{2}(:[0-9a-f]{2})+")) { + return trimmedLine.toUpperCase(Locale.ROOT); + } + } + return null; + } + + /** + * Build vpx:// URL for virt-v2v + * + * Format: + * vpx://user@vcenter/DC/cluster/host?no_verify=1 + */ + private String buildVpxUrl(RemoteInstanceTO vmwareInstance) { + + String vmName = vmwareInstance.getInstanceName(); + String vcenter = vmwareInstance.getVcenterHost(); + String username = vmwareInstance.getVcenterUsername(); + String datacenter = vmwareInstance.getDatacenterName(); + String cluster = vmwareInstance.getClusterName(); + String host = vmwareInstance.getHostName(); + + String encodedUsername = encodeUsername(username); + + StringBuilder url = new StringBuilder(); + url.append("vpx://") + .append(encodedUsername) + .append("@") + .append(vcenter) + .append("/") + .append(datacenter); + + if (StringUtils.isNotBlank(cluster)) { + url.append("/").append(cluster); + } + + if (StringUtils.isNotBlank(host)) { + url.append("/").append(host); + } + + url.append("?no_verify=1"); + + logger.info("({}) Using VPX URL: {}", vmName, url); + return url.toString(); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java new file mode 100644 index 000000000000..7cf05da9b211 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -0,0 +1,178 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.CreateImageTransferAnswer; +import org.apache.cloudstack.backup.CreateImageTransferCommand; +import org.apache.cloudstack.backup.ImageTransfer; +import org.apache.cloudstack.storage.resource.IpTablesHelper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = CreateImageTransferCommand.class) +public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private static final String IMAGE_SERVER_TLS_CERT_FILE = "/etc/cloudstack/agent/cloud.crt"; + private static final String IMAGE_SERVER_TLS_KEY_FILE = "/etc/cloudstack/agent/cloud.key"; + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private static String shellQuote(String value) { + return "'" + value.replace("'", "'\\''") + "'"; + } + + private boolean startImageServerIfNotRunning(int imageServerPort, String listenAddress, LibvirtComputingResource resource) { + final String imageServerPackageDir = resource.getImageServerPath(); + final String imageServerParentDir = new File(imageServerPackageDir).getParent(); + final String imageServerModuleName = new File(imageServerPackageDir).getName(); + final boolean tlsEnabled = resource.isImageServerTlsEnabled(); + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null && ImageServerControlSocket.isReady()) { + return true; + } + + resetService(unitName); + if (checkResult != null) { + StringBuilder systemdRunCmd = new StringBuilder(String.format( + "systemd-run --unit=%s --property=Restart=no --property=WorkingDirectory=%s /usr/bin/python3 -m %s --listen %s --port %d", + unitName, shellQuote(imageServerParentDir), imageServerModuleName, shellQuote(listenAddress), imageServerPort)); + + if (tlsEnabled) { + systemdRunCmd.append(" --tls-enabled"); + systemdRunCmd.append(" --tls-cert-file ").append(IMAGE_SERVER_TLS_CERT_FILE); + systemdRunCmd.append(" --tls-key-file ").append(IMAGE_SERVER_TLS_KEY_FILE); + } + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd.toString()); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start the Image server: %s", startResult)); + return false; + } + } + + int maxWaitSeconds = 10; + int pollIntervalMs = 1000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serverReady = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + if (ImageServerControlSocket.isReady()) { + serverReady = true; + logger.info(String.format("Image server control socket is ready (attempt %d)", attempt + 1)); + break; + } + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + if (!serverReady) { + logger.error(String.format("Image server control socket not ready within %d seconds", maxWaitSeconds)); + return false; + } + + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", imageServerPort); + IpTablesHelper.addConditionally(IpTablesHelper.INPUT_CHAIN, true, rule, + String.format("Error in opening up image server port %d", imageServerPort)); + + return true; + } + + public Answer execute(CreateImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + ImageTransfer.Backend backend = cmd.getBackend(); + + if (StringUtils.isBlank(transferId)) { + return new CreateImageTransferAnswer(cmd, false, "transferId is empty."); + } + + final Map payload = new HashMap<>(); + payload.put("backend", backend.toString()); + payload.put("idle_timeout_seconds", cmd.getIdleTimeoutSeconds()); + + if (backend == ImageTransfer.Backend.file) { + final String filePath = cmd.getFile(); + if (StringUtils.isBlank(filePath)) { + return new CreateImageTransferAnswer(cmd, false, "file path is empty for file backend."); + } + payload.put("file", filePath); + } else { + String socket = cmd.getSocket(); + final String exportName = cmd.getExportName(); + if (StringUtils.isBlank(socket)) { + return new CreateImageTransferAnswer(cmd, false, "Empty socket."); + } + if (StringUtils.isBlank(exportName)) { + return new CreateImageTransferAnswer(cmd, false, "exportName is empty."); + } + payload.put("socket", "/tmp/imagetransfer/" + socket + ".sock"); + payload.put("export", exportName); + String checkpointId = cmd.getCheckpointId(); + if (checkpointId != null) { + payload.put("export_bitmap", cmd.getCheckpointId()); + } + } + + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; + String listenAddress = resource.getImageServerListenAddress(); + if (StringUtils.isBlank(listenAddress)) { + listenAddress = resource.getPrivateIp(); + } + if (!startImageServerIfNotRunning(imageServerPort, listenAddress, resource)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to start image server."); + } + + if (!ImageServerControlSocket.registerTransfer(transferId, payload)) { + return new CreateImageTransferAnswer(cmd, false, "Failed to register transfer with image server."); + } + + final String transferScheme = resource.isImageServerTlsEnabled() ? "https" : "http"; + final String transferUrl = String.format("%s://%s:%d/images/%s", transferScheme, listenAddress, imageServerPort, transferId); + return new CreateImageTransferAnswer(cmd, true, "Image transfer prepared on KVM host.", transferId, transferUrl); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java new file mode 100644 index 000000000000..ddb84ab29cb3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java @@ -0,0 +1,79 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.Map; + +import org.apache.cloudstack.backup.DeleteVmCheckpointCommand; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = DeleteVmCheckpointCommand.class) +public class LibvirtDeleteVmCheckpointCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(DeleteVmCheckpointCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return deleteBitmapsOnDisks(cmd); + } + return deleteDomainCheckpoint(cmd); + } + + private Answer deleteDomainCheckpoint(DeleteVmCheckpointCommand cmd) { + String vmName = cmd.getVmName(); + String checkpointId = cmd.getCheckpointId(); + String virshCmd = String.format("virsh checkpoint-delete %s %s", vmName, checkpointId); + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(virshCmd); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, "Failed to delete checkpoint: " + result); + } + return new Answer(cmd, true, "Checkpoint deleted"); + } + + /** + * Stopped VM: persistent bitmaps on disk images ({@code qemu-img bitmap --remove}), matching {@link LibvirtStartBackupCommandWrapper} bitmap --add. + */ + private Answer deleteBitmapsOnDisks(DeleteVmCheckpointCommand cmd) { + String checkpointId = cmd.getCheckpointId(); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + if (diskPathUuidMap == null || diskPathUuidMap.isEmpty()) { + return new Answer(cmd, false, "No disks provided for bitmap removal"); + } + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("qemu-img"); + script.add("bitmap"); + script.add("--remove"); + script.add(diskPath); + script.add(checkpointId); + String result = script.execute(); + if (result != null) { + return new Answer(cmd, false, + "Failed to remove bitmap " + checkpointId + " from disk " + diskPath + ": " + result); + } + } + return new Answer(cmd, true, "Checkpoint bitmap removed from disks"); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java new file mode 100644 index 000000000000..3d9f6563d5eb --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFinalizeImageTransferCommandWrapper.java @@ -0,0 +1,101 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.FinalizeImageTransferCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = FinalizeImageTransferCommand.class) +public class LibvirtFinalizeImageTransferCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + private boolean stopImageServer(int imageServerPort, LibvirtComputingResource resource) { + String unitName = resource.IMAGE_SERVER_SYSTEMD_UNIT_NAME; + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + logger.info("Image server not running, resetting failed state"); + resetService(unitName); + removeFirewallRule(imageServerPort); + return true; + } + + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + logger.info("Image server {} stopped", unitName); + + removeFirewallRule(imageServerPort); + + return true; + } + + private void removeFirewallRule(int port) { + String rule = String.format("-p tcp -m state --state NEW -m tcp --dport %d -j ACCEPT", port); + Script removeScript = new Script("/bin/bash", logger); + removeScript.add("-c"); + removeScript.add(String.format("iptables -D INPUT %s || true", rule)); + String result = removeScript.execute(); + if (result != null && !result.isEmpty() && !result.contains("iptables: Bad rule")) { + logger.debug("Firewall rule removal result for port {}: {}", port, result); + } else { + logger.info("Firewall rule removed for port {} (or did not exist)", port); + } + } + + public Answer execute(FinalizeImageTransferCommand cmd, LibvirtComputingResource resource) { + final String transferId = cmd.getTransferId(); + final int imageServerPort = LibvirtComputingResource.IMAGE_SERVER_DEFAULT_PORT; + if (StringUtils.isBlank(transferId)) { + return new Answer(cmd, false, "transferId is empty."); + } + + int activeTransfers = ImageServerControlSocket.unregisterTransfer(transferId); + if (activeTransfers < 0) { + logger.warn("Could not reach image server to unregister transfer {}; assuming server is down", transferId); + stopImageServer(imageServerPort, resource); + return new Answer(cmd, true, "Image transfer finalized (server unreachable, forced stop)."); + } + + if (activeTransfers == 0) { + stopImageServer(imageServerPort, resource); + } + + return new Answer(cmd, true, "Image transfer finalized."); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 6d918435277b..ab6bdd6135d5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -52,7 +52,7 @@ public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapper { static final List STORAGE_POOL_TYPES_SUPPORTED_BY_QEMU_IMG = Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem, StoragePoolType.RBD); + StoragePoolType.Filesystem, StoragePoolType.RBD, StoragePoolType.SharedMountPoint); @Override public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 43607edc53a5..f54918bbc228 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -259,6 +259,12 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. final int migrateDowntime = libvirtComputingResource.getMigrateDowntime(); boolean isMigrateDowntimeSet = false; + final int migrateWait = libvirtComputingResource.getMigrateWait(); + logger.info("vm.migrate.wait value set to: {} secs for VM: {}", migrateWait, vmName); + + final int migratePauseAfter = libvirtComputingResource.getMigratePauseAfter(); + logger.info("vm.migrate.pauseafter value set to: {} ms for VM: {}", migratePauseAfter, vmName); + while (!executor.isTerminated()) { Thread.sleep(100); sleeptime += 100; @@ -278,8 +284,6 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. } // abort the vm migration if the job is executed more than vm.migrate.wait - final int migrateWait = libvirtComputingResource.getMigrateWait(); - logger.info("vm.migrate.wait value set to: {}for VM: {}", migrateWait, vmName); if (migrateWait > 0 && sleeptime > migrateWait * 1000) { DomainState state = null; try { @@ -306,8 +310,6 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. } // pause vm if we meet the vm.migrate.pauseafter threshold and not already paused - final int migratePauseAfter = libvirtComputingResource.getMigratePauseAfter(); - logger.info("vm.migrate.pauseafter value set to: {} for VM: {}", migratePauseAfter, vmName); if (migratePauseAfter > 0 && sleeptime > migratePauseAfter) { DomainState state = null; try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index e74923b281f6..5a7d6d2c203a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -34,6 +34,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; @ResourceWrapper(handles = ReadyCommand.class) public final class LibvirtReadyCommandWrapper extends CommandWrapper { @@ -50,6 +51,9 @@ public Answer execute(final ReadyCommand command, final LibvirtComputingResource if (libvirtComputingResource.hostSupportsInstanceConversion()) { hostDetails.put(Host.HOST_VIRTV2V_VERSION, libvirtComputingResource.getHostVirtV2vVersion()); } + hostDetails.put(Host.HOST_VDDK_SUPPORT, Boolean.toString(libvirtComputingResource.hostSupportsVddk())); + hostDetails.put(Host.HOST_VDDK_LIB_DIR, StringUtils.defaultString(libvirtComputingResource.getVddkLibDir())); + hostDetails.put(Host.HOST_VDDK_VERSION, StringUtils.defaultString(libvirtComputingResource.getVddkVersion())); if (libvirtComputingResource.hostSupportsOvfExport()) { hostDetails.put(Host.HOST_OVFTOOL_VERSION, libvirtComputingResource.getHostOvfToolVersion()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 46561a9bddfe..22dbfbdd67a2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -88,7 +88,7 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser List restoreVolumePools = command.getRestoreVolumePools(); List restoreVolumePaths = command.getRestoreVolumePaths(); Integer mountTimeout = command.getMountTimeout() * 1000; - int timeout = command.getWait(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : serverResource.getCmdsTimeout(); KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); List backupFiles = command.getBackupFiles(); @@ -270,7 +270,7 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr return replaceBlockDeviceWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, createTargetVolume, size); } - int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath)); + int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath), timeout, false); return exitValue == 0; } @@ -278,7 +278,7 @@ private boolean replaceBlockDeviceWithBackup(KVMStoragePoolManager storagePoolMg KVMStoragePool volumeStoragePool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid()); QemuImg qemu; try { - qemu = new QemuImg(timeout * 1000, true, false); + qemu = new QemuImg(timeout, true, false); String volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); KVMPhysicalDisk disk = null; if (createTargetVolume) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java index 1536984f2e85..b1f64c8c6dbb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java @@ -49,10 +49,11 @@ public Answer execute(ScaleVmCommand command, LibvirtComputingResource libvirtCo conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); Domain dm = conn.domainLookupByName(vmName); - logger.debug(String.format("Scaling %s.", scalingDetails)); + logger.debug("Scaling {}.", scalingDetails); scaleMemory(dm, newMemory, vmDefinition); scaleVcpus(dm, newVcpus, vmDefinition); updateCpuShares(dm, newCpuShares); + libvirtComputingResource.updateCpuQuotaAndPeriod(dm, vmSpec, command.getLimitCpuUseChange()); return new ScaleVmAnswer(command, true, String.format("Successfully scaled %s.", scalingDetails)); } catch (LibvirtException | CloudRuntimeException e) { @@ -74,7 +75,7 @@ protected void updateCpuShares(Domain dm, int newCpuShares) throws LibvirtExcept if (oldCpuShares < newCpuShares) { LibvirtComputingResource.setCpuShares(dm, newCpuShares); - logger.info(String.format("Successfully increased cpu_shares of VM [%s] from [%s] to [%s].", dm.getName(), oldCpuShares, newCpuShares)); + logger.info("Successfully increased cpu_shares of VM [{}] from [{}] to [{}].", dm.getName(), oldCpuShares, newCpuShares); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java new file mode 100644 index 000000000000..629fd179afb2 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -0,0 +1,300 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.StartBackupAnswer; +import org.apache.cloudstack.backup.StartBackupCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.json.JSONArray; +import org.json.JSONObject; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartBackupCommand.class) +public class LibvirtStartBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StartBackupCommand cmd, LibvirtComputingResource resource) { + if (cmd.isStoppedVM()) { + return handleStoppedVmBackup(cmd, cmd.getToCheckpointId()); + } + return handleRunningVmBackup(cmd, resource); + } + + public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + String toCheckpointId = cmd.getToCheckpointId(); + String fromCheckpointId = cmd.getFromCheckpointId(); + Long fromCheckpointCreateTime = cmd.getFromCheckpointCreateTime(); + String socket = cmd.getSocket(); + + try { + if (StringUtils.isNotBlank(fromCheckpointId)) { + Answer redefineAnswer = ensureFromCheckpointExists(cmd, fromCheckpointId, fromCheckpointCreateTime); + if (redefineAnswer != null) { + return redefineAnswer; + } + } + + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + + // Create backup XML + String backupXml = createBackupXml(cmd, fromCheckpointId, socket, resource); + String checkpointXml = createCheckpointXml(toCheckpointId); + + // Write XMLs to temp files + File backupXmlFile = File.createTempFile("backup-", ".xml"); + File checkpointXmlFile = File.createTempFile("checkpoint-", ".xml"); + + try (FileWriter writer = new FileWriter(backupXmlFile)) { + writer.write(backupXml); + } + try (FileWriter writer = new FileWriter(checkpointXmlFile)) { + writer.write(checkpointXml); + } + + // Execute virsh backup-begin + String backupCmd = String.format("virsh backup-begin %s %s --checkpointxml %s", + vmName, backupXmlFile.getAbsolutePath(), checkpointXmlFile.getAbsolutePath()); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(backupCmd); + String result = script.execute(); + + backupXmlFile.delete(); + checkpointXmlFile.delete(); + + if (result != null) { + return new StartBackupAnswer(cmd, false, "Backup begin failed: " + result); + } + + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Backup started successfully", checkpointCreateTime); + + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Error starting backup: " + e.getMessage()); + } + } + + private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromCheckpointId, Long fromCheckpointCreateTime) { + String vmName = cmd.getVmName(); + Script dumpScript = new Script("/bin/bash"); + dumpScript.add("-c"); + dumpScript.add(String.format("virsh checkpoint-dumpxml --domain %s --checkpointname %s --no-domain", + vmName, fromCheckpointId)); + if (dumpScript.execute() == null) { + return null; + } + if (fromCheckpointCreateTime == null) { + return new StartBackupAnswer(cmd, false, "From checkpoint create time is null for checkpoint " + fromCheckpointId); + } + + String redefineXml = createCheckpointXmlForRedefine(fromCheckpointId, fromCheckpointCreateTime); + File redefineFile; + try { + redefineFile = File.createTempFile("checkpoint-redefine-", ".xml"); + } catch (Exception e) { + return new StartBackupAnswer(cmd, false, "Failed to create temp file for checkpoint redefine: " + e.getMessage()); + } + try (FileWriter writer = new FileWriter(redefineFile)) { + writer.write(redefineXml); + } catch (Exception e) { + redefineFile.delete(); + return new StartBackupAnswer(cmd, false, "Failed to write checkpoint redefine XML: " + e.getMessage()); + } + String createCmd = String.format(LibvirtComputingResource.CHECKPOINT_CREATE_COMMAND, vmName, redefineFile.getAbsolutePath()); + Script createScript = new Script("/bin/bash"); + createScript.add("-c"); + createScript.add(createCmd); + String result = createScript.execute(); + redefineFile.delete(); + if (result != null) { + return new StartBackupAnswer(cmd, false, "Failed to redefine from-checkpoint " + fromCheckpointId + ": " + result); + } + return null; + } + + private String createCheckpointXmlForRedefine(String checkpointName, Long createTime) { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + xml.append(" ").append(checkpointName).append("\n"); + xml.append(" ").append(createTime).append("\n"); + xml.append(""); + return xml.toString(); + } + + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, String socket, LibvirtComputingResource resource) throws LibvirtException { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + + xml.append(String.format(" \n", socket)); + + xml.append(" \n"); + + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathLabelMap = resource.getDiskPathLabelMap(cmd.getVmName()); + Map diskPathHasFromCheckpointMap = new HashMap<>(); + if (StringUtils.isNotBlank(fromCheckpointId)) { + Domain vm = null; + try { + vm = resource.getDomain(resource.getLibvirtUtilitiesHelper().getConnection(), cmd.getVmName()); + if (vm != null) { + diskPathHasFromCheckpointMap = getVmDiskPathHasFromCheckpointMap(vm, fromCheckpointId); + } else { + logger.warn("Failed to get domain for VM [{}] while evaluating export bitmap [{}]. Falling back to full Backup", + cmd.getVmName(), fromCheckpointId); + } + } finally { + if (vm != null) { + vm.free(); + } + } + } + + for (Map.Entry entry : diskPathLabelMap.entrySet()) { + String diskPath = entry.getKey(); + if (!diskPathUuidMap.containsKey(diskPath)) { + continue; + } + String diskName = entry.getValue(); + String export = diskPathUuidMap.get(diskPath); + String scratchFile = "/var/tmp/scratch-" + export + ".qcow2"; + xml.append(" \n"); + xml.append(" \n"); + xml.append(" \n"); + } + + xml.append(" \n"); + xml.append(""); + + return xml.toString(); + } + + private String createCheckpointXml(String checkpointId) { + return "\n" + + " " + checkpointId + "\n" + + ""; + } + + private Answer handleStoppedVmBackup(StartBackupCommand cmd, String toCheckpointId) { + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + Map diskPathPassphraseMap = cmd.getDiskPathPassphraseMap(); + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("qemu-img"); + byte[] passphrase = diskPathPassphraseMap.get(diskPath); + + script.add("bitmap"); + script.add("--add"); + + if (passphrase != null && passphrase.length > 0) { + KeyFile srcKey; + try { + srcKey = new KeyFile(passphrase); + } catch (IOException ex) { + return new StartBackupAnswer(cmd, false, "Failed to create KeyFile while adding bitmap " + toCheckpointId + " to disk " + diskPath); + } + script.add("--object"); + script.add(String.format("secret,id=sec0,file=%s", srcKey)); + script.add("--image-opts"); + script.add(String.format("driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", diskPath)); + } else { + script.add(diskPath); + } + + script.add(toCheckpointId); + String result = script.execute(); + if (result != null) { + return new StartBackupAnswer(cmd, false, + "Failed to add bitmap " + toCheckpointId + " to disk " + diskPath + ": " + result); + } + } + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Stopped VM backup: checkpoint bitmap added successfully", + checkpointCreateTime); + } + + private long getCheckpointCreateTime() { + return System.currentTimeMillis() / 1000; + } + + private Map getVmDiskPathHasFromCheckpointMap(Domain vm, String fromCheckpointId) throws LibvirtException { + Map diskPathHasFromCheckpointMap = new HashMap<>(); + String queryBlock = vm.qemuMonitorCommand(QemuCommand.buildQemuCommand("query-block", null), 0); + JSONObject response = new JSONObject(queryBlock); + JSONArray blocks = response.optJSONArray("return"); + if (blocks == null) { + logger.warn("Couldn't get bitmap information for the VM [{}]. Falling back to full Backup", vm.getName()); + return diskPathHasFromCheckpointMap; + } + for (int i = 0; i < blocks.length(); i++) { + JSONObject block = blocks.getJSONObject(i); + JSONObject inserted = block.optJSONObject("inserted"); + if (inserted == null) { + continue; + } + String file = inserted.optString("file"); + if (StringUtils.isBlank(file)) { + continue; + } + JSONArray dirtyBitmaps = inserted.optJSONArray("dirty-bitmaps"); + boolean hasFromCheckpointBitmap = false; + if (dirtyBitmaps != null) { + for (int j = 0; j < dirtyBitmaps.length(); j++) { + JSONObject dirtyBitmap = dirtyBitmaps.optJSONObject(j); + if (dirtyBitmap == null) { + continue; + } + String bitmapName = dirtyBitmap.optString("name"); + if (fromCheckpointId.equals(bitmapName)) { + hasFromCheckpointBitmap = true; + break; + } + } + } + diskPathHasFromCheckpointMap.put(file, hasFromCheckpointBitmap); + } + return diskPathHasFromCheckpointMap; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java new file mode 100644 index 000000000000..661344677640 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -0,0 +1,197 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.io.File; +import java.io.IOException; + +import org.apache.cloudstack.backup.StartNBDServerAnswer; +import org.apache.cloudstack.backup.StartNBDServerCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.json.JSONArray; +import org.json.JSONObject; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StartNBDServerCommand.class) +public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private String createSystemdRunCmd(StartNBDServerCommand cmd, String unitName, String volumePath, String exportName, String socket) throws IOException { + String socketName = "/tmp/imagetransfer/" + socket + ".sock"; + + String bitmapArg = ""; + if (StringUtils.isNotBlank(cmd.getFromCheckpointId()) + && isBitmapPresentOnDisk(volumePath, cmd.getFromCheckpointId())) { + bitmapArg = "-B " + cmd.getFromCheckpointId(); + } + + byte[] passphrase = cmd.getPassphrase(); // may be null + String readOnlyArg = cmd.getDirection().equals("download") ? "--read-only" : ""; + String imageArg = volumePath; + String secretArg = ""; + if (passphrase != null && passphrase.length > 0) { + KeyFile srcKey = new KeyFile(passphrase); + secretArg = String.format("--object secret,id=sec0,file=%s", srcKey); + imageArg = String.format("--image-opts driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", volumePath); + } + + // --persistent: Don't stop the service when the last client disconnects. + // --shared=NUM: Allow up to NUM clients to share the device (default 1), 0 for unlimited. Number of parallel connections is managed by the image server. + String systemdRunCmd = String.format( + "systemd-run --unit=%s --property=Restart=no qemu-nbd %s " + + "--export-name %s --socket %s --persistent --shared=0 %s %s %s", + unitName, + secretArg, + exportName, + socketName, + bitmapArg, + readOnlyArg, + imageArg + ); + return systemdRunCmd; + } + + @Override + public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { + String volumePath = cmd.getVolumePath(); + String socket = cmd.getSocket(); + String exportName = cmd.getExportName(); + String transferId = cmd.getTransferId(); + + if (StringUtils.isBlank(volumePath)) { + return new StartNBDServerAnswer(cmd, false, "Volume path is required for the nbd server"); + } + if (StringUtils.isBlank(exportName)) { + return new StartNBDServerAnswer(cmd, false, "Export name is required for the nbd server"); + } + if (StringUtils.isBlank(socket)) { + return new StartNBDServerAnswer(cmd, false, "Socket is required for the nbd server"); + } + + String unitName = "qemu-nbd-" + transferId.hashCode(); + + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult == null) { + return new StartNBDServerAnswer(cmd, false, "A qemu-nbd service is already running on the port."); + } + + File dir = new File("/tmp/imagetransfer"); + if (!dir.exists()) { + dir.mkdirs(); + } + + String systemdRunCmd = ""; + try { + systemdRunCmd = createSystemdRunCmd(cmd, unitName, volumePath, exportName, socket); + } catch (IOException e) { + logger.error("Failed to create the KeyFile for qemu-nbd service", e); + return new StartNBDServerAnswer(cmd, false, "Failed to create systemd run command for qemu-nbd service: " + e.getMessage()); + } + + Script startScript = new Script("/bin/bash", logger); + startScript.add("-c"); + startScript.add(systemdRunCmd); + String startResult = startScript.execute(); + + if (startResult != null) { + logger.error(String.format("Failed to start qemu-nbd service: %s", startResult)); + return new StartNBDServerAnswer(cmd, false, "Failed to start qemu-nbd service: " + startResult); + } + + // Wait with timeout until the service is up + int maxWaitSeconds = 20; + int pollIntervalMs = 5000; + int maxAttempts = (maxWaitSeconds * 1000) / pollIntervalMs; + boolean serviceActive = false; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + try { + Thread.sleep(pollIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new StartNBDServerAnswer(cmd, false, "Interrupted while waiting for qemu-nbd service to start"); + } + Script verifyScript = new Script("/bin/bash", logger); + verifyScript.add("-c"); + verifyScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String verifyResult = verifyScript.execute(); + if (verifyResult == null) { + serviceActive = true; + logger.info(String.format("qemu-nbd service %s is now active (attempt %d)", unitName, attempt + 1)); + break; + } + } + + if (!serviceActive) { + logger.error(String.format("qemu-nbd service %s failed to become active within %d seconds", unitName, maxWaitSeconds)); + return new StartNBDServerAnswer(cmd, false, + String.format("qemu-nbd service failed to start within %d seconds", maxWaitSeconds)); + } + + String transferUrl = String.format("nbd+unix:///%s", cmd.getSocket()); + return new StartNBDServerAnswer(cmd, true, "qemu-nbd service started for upload", + transferId, transferUrl); + } + + private boolean isBitmapPresentOnDisk(String volumePath, String fromCheckpointId) { + String qemuImgInfo = Script.runBashScriptIgnoreExitValue( + String.format("qemu-img info --output=json %s", volumePath), 0); + if (StringUtils.isBlank(qemuImgInfo)) { + logger.warn("Unable to read qemu-img info output for disk path [{}].", volumePath); + return false; + } + try { + JSONObject info = new JSONObject(qemuImgInfo); + JSONObject formatSpecific = info.optJSONObject("format-specific"); + if (formatSpecific == null) { + return false; + } + JSONObject formatData = formatSpecific.optJSONObject("data"); + if (formatData == null) { + return false; + } + JSONArray bitmaps = formatData.optJSONArray("bitmaps"); + if (bitmaps == null) { + return false; + } + for (int i = 0; i < bitmaps.length(); i++) { + JSONObject bitmap = bitmaps.optJSONObject(i); + if (bitmap == null) { + continue; + } + if (fromCheckpointId.equals(bitmap.optString("name"))) { + return true; + } + } + } catch (Exception e) { + logger.warn("Failed to parse qemu-img info output for disk path [{}].", volumePath, e); + } + return false; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java new file mode 100644 index 000000000000..1185d89bc0b3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopBackupCommandWrapper.java @@ -0,0 +1,69 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopBackupAnswer; +import org.apache.cloudstack.backup.StopBackupCommand; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.libvirt.Connect; +import org.libvirt.Domain; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopBackupCommand.class) +public class LibvirtStopBackupCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + @Override + public Answer execute(StopBackupCommand cmd, LibvirtComputingResource resource) { + String vmName = cmd.getVmName(); + + try { + Connect conn = LibvirtConnection.getConnection(); + Domain dm = conn.domainLookupByName(vmName); + + if (dm == null) { + return new StopBackupAnswer(cmd, false, "Domain not found: " + vmName); + } + + // Execute virsh domjobabort + String abortCmd = String.format("virsh domjobabort %s", vmName); + + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(abortCmd); + String result = script.execute(); + + if (result != null && !result.isEmpty()) { + // Job abort may fail if no job is running, which is acceptable + logger.debug("domjobabort result: " + result); + } + + return new StopBackupAnswer(cmd, true, "Backup stopped successfully"); + + } catch (Exception e) { + return new StopBackupAnswer(cmd, false, "Error stopping backup: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java new file mode 100644 index 000000000000..57c7ebb706bc --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopNBDServerCommandWrapper.java @@ -0,0 +1,72 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.backup.StopNBDServerCommand; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = StopNBDServerCommand.class) +public class LibvirtStopNBDServerCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + + private void resetService(String unitName) { + Script resetScript = new Script("/bin/bash", logger); + resetScript.add("-c"); + resetScript.add(String.format("systemctl reset-failed %s || true", unitName)); + resetScript.execute(); + } + + @Override + public Answer execute(StopNBDServerCommand cmd, LibvirtComputingResource resource) { + try { + String unitName = "qemu-nbd-" + cmd.getTransferId().hashCode(); + + // Check if the service is running + Script checkScript = new Script("/bin/bash", logger); + checkScript.add("-c"); + checkScript.add(String.format("systemctl is-active --quiet %s", unitName)); + String checkResult = checkScript.execute(); + if (checkResult != null) { + // Service is not running, but still reset-failed to clear any stale state + logger.info(String.format("qemu-nbd service %s is not running, resetting failed state", unitName)); + resetService(unitName); + return new Answer(cmd, true, "Image transfer finalized"); + } + + // Stop the systemd service + Script stopScript = new Script("/bin/bash", logger); + stopScript.add("-c"); + stopScript.add(String.format("systemctl stop %s", unitName)); + stopScript.execute(); + resetService(unitName); + + return new Answer(cmd, true, "Image transfer finalized"); + + } catch (Exception e) { + logger.error("Error finalizing image transfer for upload", e); + return new Answer(cmd, false, "Error finalizing image transfer: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 11fa605908a6..42953aa9f835 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -52,6 +52,7 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir List volumePools = command.getVolumePools(); final List volumePaths = command.getVolumePaths(); KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout(); List diskPaths = new ArrayList<>(); if (Objects.nonNull(volumePaths)) { @@ -81,7 +82,7 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir "-d", diskPaths.isEmpty() ? "" : String.join(",", diskPaths) }); - Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + Pair result = Script.executePipedCommands(commands, timeout); if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java index f5bfd898a4f4..8cf6de68f955 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java @@ -208,12 +208,12 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 8dd2116e1235..3e35ed9476b1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -33,35 +33,33 @@ public interface KVMStoragePool { - public static final long HeartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); - public static final long HeartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); - public static final long HeartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); - public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); - public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); + long HeartBeatUpdateTimeoutInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); + long HeartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + long HeartBeatCheckerTimeoutInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); - public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { + default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase); } - public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); + KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); - public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); + KVMPhysicalDisk createPhysicalDisk(String volumeUuid, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); - public boolean connectPhysicalDisk(String volumeUuid, Map details); + boolean connectPhysicalDisk(String volumeUuid, Map details); - public KVMPhysicalDisk getPhysicalDisk(String volumeUuid); + KVMPhysicalDisk getPhysicalDisk(String volumeUuid); - public boolean disconnectPhysicalDisk(String volumeUuid); + boolean disconnectPhysicalDisk(String volumeUuid); - public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format); + boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format); - public List listPhysicalDisks(); + List listPhysicalDisks(); - public String getUuid(); + String getUuid(); - public long getCapacity(); + long getCapacity(); - public long getUsed(); + long getUsed(); default Long getCapacityIops() { return null; @@ -71,51 +69,51 @@ default Long getUsedIops() { return null; } - public long getAvailable(); + long getAvailable(); - public boolean refresh(); + boolean refresh(); - public boolean isExternalSnapshot(); + boolean isExternalSnapshot(); - public String getLocalPath(); + String getLocalPath(); - public String getSourceHost(); + String getSourceHost(); - public String getSourceDir(); + String getSourceDir(); - public int getSourcePort(); + int getSourcePort(); - public String getAuthUserName(); + String getAuthUserName(); - public String getAuthSecret(); + String getAuthSecret(); - public StoragePoolType getType(); + StoragePoolType getType(); - public boolean delete(); + boolean delete(); PhysicalDiskFormat getDefaultFormat(); - public boolean createFolder(String path); + boolean createFolder(String path); - public boolean supportsConfigDriveIso(); + boolean supportsConfigDriveIso(); - public Map getDetails(); + Map getDetails(); default String getLocalPathFor(String relativePath) { return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); } - public boolean isPoolSupportHA(); + boolean isPoolSupportHA(); - public String getHearthBeatPath(); + String getHearthBeatPath(); - public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); + String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); - public String getStorageNodeId(); + String getStorageNodeId(); - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host); + Boolean hasHeartBeat(HAStoragePool pool, HostTO host); - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration); + Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration); default LibvirtVMDef.DiskDef.BlockIOSize getSupportedLogicalBlockSize() { return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 6665cf625e2f..35cc864268c3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -289,6 +289,7 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean if (pool instanceof LibvirtStoragePool) { addPoolDetails(uuid, (LibvirtStoragePool) pool); + ((LibvirtStoragePool) pool).setType(type); } return pool; @@ -390,6 +391,9 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); + if (pool instanceof LibvirtStoragePool) { + ((LibvirtStoragePool) pool).setType(type); + } // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 030d9747d6cd..f95ebff5326f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -185,6 +185,8 @@ public class KVMStorageProcessor implements StorageProcessor { private int incrementalSnapshotTimeout; + private int incrementalSnapshotRetryRebaseWait; + private static final String CHECKPOINT_XML_TEMP_DIR = "/tmp/cloudstack/checkpointXMLs"; private static final String BACKUP_XML_TEMP_DIR = "/tmp/cloudstack/backupXMLs"; @@ -252,6 +254,7 @@ public boolean configure(final String name, final Map params) th _cmdsTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CMDS_TIMEOUT) * 1000; incrementalSnapshotTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000; + incrementalSnapshotRetryRebaseWait = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT) * 1000; return true; } @@ -2044,7 +2047,7 @@ private void waitForBackup(String vmName) throws CloudRuntimeException { try { Thread.sleep(10000); } catch (InterruptedException e) { - throw new CloudRuntimeException(e); + logger.trace("Thread that was tracking the progress for backup of VM [{}] was interrupted. Ignoring.", vmName); } } @@ -2093,8 +2096,25 @@ protected void rebaseSnapshot(SnapshotObjectTO snapshotObjectTO, KVMStoragePool QemuImg qemuImg = new QemuImg(wait); qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); } catch (LibvirtException | QemuImgException e) { - logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); - throw new CloudRuntimeException(e); + if (!StringUtils.contains(e.getMessage(), "Is another process using the image")) { + logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + retryRebase(snapshotName, wait, e, snapshotFile, parentSnapshotFile); + } + } + + private void retryRebase(String snapshotName, int wait, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) { + logger.warn("Libvirt still has not released the lock, will wait [{}] milliseconds and try again later.", incrementalSnapshotRetryRebaseWait); + try { + Thread.sleep(incrementalSnapshotRetryRebaseWait); + QemuImg qemuImg = new QemuImg(wait); + qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); + } catch (LibvirtException | QemuImgException | InterruptedException ex) { + logger.error("Unable to rebase snapshot [{}].", snapshotName, ex); + CloudRuntimeException cre = new CloudRuntimeException(ex); + cre.addSuppressed(e); + throw cre; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index ab39f7bc6ffd..910f0eb15e0b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -31,6 +31,7 @@ import com.cloud.agent.api.to.HostTO; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; @@ -320,29 +321,38 @@ public void setDetails(Map details) { @Override public boolean isPoolSupportHA() { - return type == StoragePoolType.NetworkFilesystem; + return HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type); } public String getHearthBeatPath() { - if (type == StoragePoolType.NetworkFilesystem) { + if (StoragePoolType.NetworkFilesystem.equals(type)) { String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - return Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); + String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); + if (scriptPath == null) { + throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); + } + return scriptPath; + } else if (StoragePoolType.SharedMountPoint.equals(type)) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); + if (scriptPath == null) { + throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); + } + return scriptPath; } return null; } public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { - Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeout, logger); + Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); cmd.add("-i", primaryStoragePool.getPoolIp()); cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); cmd.add("-m", primaryStoragePool.getMountDestPath()); if (hostValidation) { cmd.add("-h", hostPrivateIp); - } - - if (!hostValidation) { + } else { cmd.add("-c"); } @@ -360,54 +370,58 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { - boolean validResult = false; + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { String hostIp = host.getPrivateNetwork().getIp(); - Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeout, logger); + Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); cmd.add("-i", pool.getPoolIp()); cmd.add("-p", pool.getPoolMountSourcePath()); cmd.add("-m", pool.getMountDestPath()); cmd.add("-h", hostIp); cmd.add("-r"); - cmd.add("-t", String.valueOf(HeartBeatUpdateFreq / 1000)); + cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); - logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, - pool.getPoolIp())); + logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned [%s]. [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), - result, parsedLine, hostIp)); + logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + return false; } else { - validResult = true; + logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString()); + return true; } - return validResult; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + String hostIp = host.getPrivateNetwork().getIp(); Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); cmd.add("-i", pool.getPoolIp()); cmd.add("-p", pool.getPoolMountSourcePath()); cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", host.getPrivateNetwork().getIp()); + cmd.add("-h", hostIp); cmd.add("-u", volumeUUIDListString); - cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000))); + cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); cmd.add("-d", String.valueOf(duration)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); - logger.debug(String.format("Checking heart beat with KVMHAVMActivityChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, pool.getPoolIp())); + logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn(String.format("Checking heart beat with KVMHAVMActivityChecker command [%s] returned [%s]. It is [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), result, parsedLine, host.getPrivateNetwork().getIp())); + logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); return false; } else { + logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString()); return true; } } + + public void setType(StoragePoolType type) { + this.type = type; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java index 229481b1f795..df9986c99194 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java @@ -225,13 +225,13 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, - String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java index e8243c3f7cfe..fc512ff94a94 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java @@ -236,12 +236,12 @@ public String getStorageNodeId() { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java index 59ea720328f5..3fbb5340fcce 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java @@ -19,38 +19,37 @@ import org.apache.cloudstack.framework.config.ConfigKey; -public class KVMHAConfig { +public interface KVMHAConfig { - public static final ConfigKey KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10", + ConfigKey KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10", "The maximum length of time, in seconds, expected for an health check to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60", + ConfigKey KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60", "The maximum length of time, in seconds, expected for an activity check to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60", + ConfigKey KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60", "The interval, in seconds, between activity checks.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10", + ConfigKey KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10", "The maximum number of activity check attempts to perform before deciding to recover or degrade a resource.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7", + ConfigKey KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7", "The activity check failure threshold ratio. This is used with the activity check maximum attempts for deciding to recover or degrade a resource. For most environments, please keep this value above 0.5.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300", + ConfigKey KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300", "The maximum length of time, in seconds, a resource can be in degraded state where only health checks are performed.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60", + ConfigKey KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60", "The maximum length of time, in seconds, expected for a recovery operation to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600", + ConfigKey KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600", "The maximum length of time, in seconds, to wait for a resource to recover.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1", + ConfigKey KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1", "The maximum recovery attempts to be made for a resource, after which the resource is fenced. The recovery counter resets when a health check passes for a resource.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", + ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster); - } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index b937be5265b7..f0b5cfc337de 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -68,17 +68,18 @@ public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACh @Override public boolean recover(Host r) throws HARecoveryException { + logger.debug("Recover the host {}", r); try { - if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ + if (outOfBandManagementService.isOutOfBandManagementEnabled(r)) { final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); return resp.getSuccess(); } else { logger.warn("OOBM recover operation failed for the host {}", r); return false; } - } catch (Exception e){ + } catch (Exception e) { logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage()); - throw new HARecoveryException(String.format(" OOBM service is not configured or enabled for this host %s", r), e); + throw new HARecoveryException(String.format("OOBM service is not configured or enabled for this host %s", r), e); } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java index 31f87d7e0442..af7441c4fd29 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java @@ -19,6 +19,7 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckOnHostAnswer; import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.dc.dao.ClusterDao; @@ -61,7 +62,7 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Inject private AgentManager agentMgr; @Inject - private PrimaryDataStoreDao storagePool; + private PrimaryDataStoreDao storagePoolDao; @Inject private StorageManager storageManager; @Inject @@ -70,11 +71,11 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Override public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException { try { - return isVMActivityOnHost(r, suspectTime); + return hasVMActivityOnHost(r, suspectTime); } catch (HACheckerException e) { - //Re-throwing the exception to avoid poluting the 'HACheckerException' already thrown + //Re-throwing the exception to avoid polluting the 'HACheckerException' already thrown throw e; - } catch (Exception e){ + } catch (Exception e) { String message = String.format("Operation timed out, probably the %s is not reachable.", r.toString()); logger.warn(message, e); throw new HACheckerException(message, e); @@ -83,82 +84,115 @@ public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException @Override public boolean isHealthy(Host r) { - return isAgentActive(r); + return isHostAgentUp(r); } - private boolean isAgentActive(Host agent) { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { - throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType())); + private boolean isHostAgentUp(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", host.getHypervisorType())); + } + + Status hostStatus = getHostAgentStatus(host); + + logger.debug("{} has the status [{}].", host.toString(), hostStatus); + return hostStatus == Status.Up; + } + + public Status getHostAgentStatus(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + return null; + } + + Status hostStatusFromItself = checkHostStatusWithSameHost(host); + if (hostStatusFromItself == Status.Up) { + return Status.Up; } - Status hostStatus = Status.Unknown; - Status neighbourStatus = Status.Unknown; - final CheckOnHostCommand cmd = new CheckOnHostCommand(agent, HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value()); + + Status hostStatusFromNeighbour = checkHostStatusWithNeighbourHosts(host); + Status hostStatus = hostStatusFromItself; + if (hostStatusFromNeighbour == Status.Up && (hostStatusFromItself == Status.Disconnected || hostStatusFromItself == Status.Down)) { + hostStatus = Status.Disconnected; + } + if (hostStatusFromNeighbour == Status.Down && (hostStatusFromItself == Status.Disconnected || hostStatusFromItself == Status.Down)) { + hostStatus = Status.Down; + } + + logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, host); + return hostStatus; + } + + private Status checkHostStatusWithSameHost(Host host) { + Status hostStatus; + boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); + final CheckOnHostCommand cmd = new CheckOnHostCommand(host, reportFailureIfOneStorageIsDown); try { - logger.debug(String.format("Checking %s status...", agent.toString())); - Answer answer = agentMgr.easySend(agent.getId(), cmd); + logger.debug("Checking {} status...", host.toString()); + Answer answer = agentMgr.easySend(host.getId(), cmd); if (answer != null) { - hostStatus = answer.getResult() ? Status.Down : Status.Up; - logger.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus)); - - if ( hostStatus == Status.Up ){ - return true; + if (answer.getResult()) { + hostStatus = ((CheckOnHostAnswer)answer).isAlive() ? Status.Up : Status.Down; + } else { + logger.debug("{} is not active according to itself, details: {}.", host.toString(), answer.getDetails()); + hostStatus = Status.Down; } - } - else { - logger.debug(String.format("Setting %s to \"Disconnected\" status.", agent.toString())); + logger.debug("{} has the status [{}].", host.toString(), hostStatus); + } else { + logger.debug("Setting {} to \"Disconnected\" status.", host.toString()); hostStatus = Status.Disconnected; } } catch (Exception e) { - logger.warn(String.format("Failed to send command CheckOnHostCommand to %s.", agent.toString()), e); + logger.warn("Failed to send command CheckOnHostCommand to {}.", host.toString(), e); + hostStatus = Status.Disconnected; } - List neighbors = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); + return hostStatus; + } + + private Status checkHostStatusWithNeighbourHosts(Host host) { + Status hostStatusFromNeighbour = Status.Unknown; + boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); + final CheckOnHostCommand cmd = new CheckOnHostCommand(host, reportFailureIfOneStorageIsDown); + List neighbors = resourceManager.listHostsInClusterByStatus(host.getClusterId(), Status.Up); for (HostVO neighbor : neighbors) { - if (neighbor.getId() == agent.getId() || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { + if (neighbor.getId() == host.getId() + || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { continue; } try { - logger.debug(String.format("Investigating %s via neighbouring %s.", agent.toString(), neighbor.toString())); - + logger.debug("Investigating {} via neighboring {}.", host.toString(), neighbor.toString()); Answer answer = agentMgr.easySend(neighbor.getId(), cmd); if (answer != null) { - neighbourStatus = answer.getResult() ? Status.Down : Status.Up; - - logger.debug(String.format("Neighbouring %s returned status [%s] for the investigated %s.", neighbor.toString(), neighbourStatus, agent.toString())); - - if (neighbourStatus == Status.Up) { - break; + if (answer.getResult()) { + hostStatusFromNeighbour = ((CheckOnHostAnswer)answer).isAlive() ? Status.Up : Status.Down; + logger.debug("Neighboring {} returned status [{}] for the investigated {}.", neighbor.toString(), hostStatusFromNeighbour, host.toString()); + if (hostStatusFromNeighbour == Status.Up) { + return hostStatusFromNeighbour; + } + } else { + logger.debug("{} is not active according to neighbor {}, details: {}.", host.toString(), neighbor.toString(), answer.getDetails()); } } else { - logger.debug(String.format("Neighbouring %s is Disconnected.", neighbor.toString())); + logger.debug("Neighboring {} is Disconnected.", neighbor.toString()); } } catch (Exception e) { - logger.warn(String.format("Failed to send command CheckOnHostCommand to %s.", neighbor.toString()), e); + logger.warn("Failed to send command CheckOnHostCommand to neighbor {}.", neighbor.toString(), e); } } - if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Disconnected; - } - if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Down; - } - logger.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus)); - - return hostStatus == Status.Up; + return hostStatusFromNeighbour; } - private boolean isVMActivityOnHost(Host agent, DateTime suspectTime) throws HACheckerException { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { - throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType())); + private boolean hasVMActivityOnHost(Host host, DateTime suspectTime) throws HACheckerException { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", host.getHypervisorType())); } boolean activityStatus = true; - HashMap> poolVolMap = getVolumeUuidOnHost(agent); - for (StoragePool pool : poolVolMap.keySet()) { - activityStatus = verifyActivityOfStorageOnHost(poolVolMap, pool, agent, suspectTime, activityStatus); + HashMap> poolVolumeMap = getStoragePoolAndVolumeInfoOnHost(host); + for (StoragePool pool : poolVolumeMap.keySet()) { + activityStatus = verifyActivityOfStorageOnHost(poolVolumeMap, pool, host, suspectTime, activityStatus); if (!activityStatus) { - logger.warn("It seems that the storage pool [{}] does not have activity on {}.", pool, agent); + logger.warn("It seems that the storage pool [{}] does not have activity on {}.", pool, host); break; } } @@ -166,66 +200,64 @@ private boolean isVMActivityOnHost(Host agent, DateTime suspectTime) throws HACh return activityStatus; } - protected boolean verifyActivityOfStorageOnHost(HashMap> poolVolMap, StoragePool pool, Host agent, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException { + protected boolean verifyActivityOfStorageOnHost(HashMap> poolVolMap, StoragePool pool, Host host, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException { List volume_list = poolVolMap.get(pool); - final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime); + final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(host, pool, volume_list, suspectTime); - logger.debug("Checking VM activity for {} on storage pool [{}].", agent.toString(), pool); + logger.debug("Checking VM activity for {} on storage pool [{}].", host.toString(), pool); try { - Answer answer = storageManager.sendToPool(pool, getNeighbors(agent), cmd); - + Answer answer = storageManager.sendToPool(pool, getNeighbors(host), cmd); if (answer != null) { activityStatus = !answer.getResult(); - logger.debug("{} {} activity on storage pool [{}]", agent.toString(), activityStatus ? "has" : "does not have", pool); + logger.debug("{} {} activity on storage pool [{}]", host.toString(), activityStatus ? "has" : "does not have", pool); } else { - String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", agent.toString(), pool); + String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", host.toString(), pool); logger.debug(message); throw new IllegalStateException(message); } - } catch (StorageUnavailableException e){ - String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool, agent); + } catch (StorageUnavailableException e) { + String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool, host); logger.warn(message, e); throw new HACheckerException(message, e); } return activityStatus; } - private HashMap> getVolumeUuidOnHost(Host agent) { - List vm_list = vmInstanceDao.listByHostId(agent.getId()); - List volume_list = new ArrayList(); - for (VirtualMachine vm : vm_list) { + private HashMap> getStoragePoolAndVolumeInfoOnHost(Host host) { + List vmListOnHost = vmInstanceDao.listByHostId(host.getId()); + List volumeListOnHost = new ArrayList<>(); + for (VirtualMachine vm : vmListOnHost) { logger.debug("Retrieving volumes of VM [{}]...", vm); - List vm_volume_list = volumeDao.findByInstance(vm.getId()); - volume_list.addAll(vm_volume_list); + List volumeListOfVM = volumeDao.findByInstance(vm.getId()); + volumeListOnHost.addAll(volumeListOfVM); } - HashMap> poolVolMap = new HashMap>(); - for (Volume vol : volume_list) { - StoragePool sp = storagePool.findById(vol.getPoolId()); - logger.debug("Retrieving storage pool [{}] of volume [{}]...", sp, vol); - if (!poolVolMap.containsKey(sp)) { - List list = new ArrayList(); - list.add(vol); + HashMap> poolVolumeMap = new HashMap<>(); + for (Volume volume : volumeListOnHost) { + StoragePool pool = storagePoolDao.findById(volume.getPoolId()); + logger.debug("Retrieving storage pool [{}] of volume [{}]...", pool, volume); + if (!poolVolumeMap.containsKey(pool)) { + List volList = new ArrayList<>(); + volList.add(volume); - poolVolMap.put(sp, list); + poolVolumeMap.put(pool, volList); } else { - poolVolMap.get(sp).add(vol); + poolVolumeMap.get(pool).add(volume); } } - return poolVolMap; + return poolVolumeMap; } - public long[] getNeighbors(Host agent) { - List neighbors = new ArrayList(); - List cluster_hosts = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); - logger.debug("Retrieving all \"Up\" hosts from cluster [{}]...", clusterDao.findById(agent.getClusterId())); - for (HostVO host : cluster_hosts) { - if (host.getId() == agent.getId() || (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { + public long[] getNeighbors(Host host) { + List neighbors = new ArrayList<>(); + List clusterHosts = resourceManager.listHostsInClusterByStatus(host.getClusterId(), Status.Up); + logger.debug("Retrieving all \"Up\" hosts from cluster [{}]...", clusterDao.findById(host.getClusterId())); + for (HostVO clusterHost : clusterHosts) { + if (clusterHost.getId() == host.getId() || (clusterHost.getHypervisorType() != Hypervisor.HypervisorType.KVM && clusterHost.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { continue; } - neighbors.add(host.getId()); + neighbors.add(clusterHost.getId()); } return ArrayUtils.toPrimitive(neighbors.toArray(new Long[neighbors.size()])); } - } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index b96295240076..b3bdafb73751 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -3133,7 +3133,7 @@ public void testCheckOnHostCommand() { assertNotNull(wrapper); final Answer answer = wrapper.execute(command, libvirtComputingResourceMock); - assertTrue(answer.getResult()); + assertFalse(answer.getResult()); verify(libvirtComputingResourceMock, times(1)).getMonitor(); } @@ -5642,35 +5642,45 @@ public void testAddExtraConfigComponentNotEmptyExtraConfig() { Mockito.verify(vmDef, times(1)).addComp(any()); } - public void validateGetCurrentMemAccordingToMemBallooningWithoutMemBalooning(){ + @Test + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryWithoutMemBallooning(){ VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); - Mockito.when(vmTo.getType()).thenReturn(Type.User); LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); libvirtComputingResource.noMemBalloon = true; - long maxMemory = 2048; + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; - long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, maxMemory); - Assert.assertEquals(maxMemory, currentMemory); - Mockito.verify(vmTo, Mockito.times(0)).getMinRam(); + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(requestedMemory, currentMemory); } @Test - public void validateGetCurrentMemAccordingToMemBallooningWithtMemBalooning(){ + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryWithMemoryBallooning(){ LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); libvirtComputingResource.noMemBalloon = false; - long maxMemory = 2048; - long minMemory = ByteScaleUtils.mebibytesToBytes(64); - VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); Mockito.when(vmTo.getType()).thenReturn(Type.User); - Mockito.when(vmTo.getMinRam()).thenReturn(minMemory); + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; - long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, maxMemory); - Assert.assertEquals(ByteScaleUtils.bytesToKibibytes(minMemory), currentMemory); - Mockito.verify(vmTo).getMinRam(); + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(minMemory, currentMemory); } + @Test + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryForSystemVms() { + LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); + libvirtComputingResource.noMemBalloon = false; + + VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); + Mockito.when(vmTo.getType()).thenReturn(Type.SecondaryStorageVm); + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; + + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(requestedMemory, currentMemory); + } @Test public void validateCreateGuestResourceDefWithVcpuMaxLimit(){ LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); @@ -7234,4 +7244,82 @@ public void getInterfaceTestInvalidMacAddressThrowCloudRuntimeException() { libvirtComputingResourceSpy.getInterface(connMock, vmName, invalidMacAddress); } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreNotUpdatedWhenLibvirtVersionIsLessThanTheMinimum() throws LibvirtException { + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 8999; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, null, false); + Mockito.verify(domainMock, Mockito.never()).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreNotUpdatedWhenThereIsNoCapCapChangeAndNoCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, false); + Mockito.verify(domainMock, Mockito.never()).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertQuotaIsRemovedWhenThereIsCpuCapChangeAndNoCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, true); + Mockito.verify(domainMock, Mockito.times(1)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreUpdatedWhenThereIsNotCpuCapChangeAndCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); + double cpuQuotaPercentage = 0.03; + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(cpuQuotaPercentage); + Mockito.doReturn(new Pair<>(1000, 300L)).when(libvirtComputingResourceSpy).getPeriodAndQuota(cpuQuotaPercentage); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, false); + Mockito.verify(domainMock, Mockito.times(2)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreUpdatedWhenThereIsCpuCapChangeAndCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); + double cpuQuotaPercentage = 0.03; + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(cpuQuotaPercentage); + Mockito.doReturn(new Pair<>(1000, 300L)).when(libvirtComputingResourceSpy).getPeriodAndQuota(cpuQuotaPercentage); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, true); + Mockito.verify(domainMock, Mockito.times(2)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void getPeriodAndQuotaTestAssertQuotaIsEqualToPeriodMultipliedByQuotaPercentage() { + double cpuQuotaPercentage = 0.3; + int expectedPeriod = CpuTuneDef.DEFAULT_PERIOD; + long expectedQuota = (long) (expectedPeriod * cpuQuotaPercentage); + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void getPeriodAndQuotaTestQuotaIsEqualToMinimumWhenRequired() { + double cpuQuotaPercentage = 0.03; + long expectedQuota = CpuTuneDef.MIN_QUOTA; + int expectedPeriod = (int) ((double) expectedQuota / cpuQuotaPercentage); + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void getPeriodAndQuotaTestPeriodIsEqualToMaximumWhenRequired() { + double cpuQuotaPercentage = 0.0003; + long expectedQuota = CpuTuneDef.MIN_QUOTA; + int expectedPeriod = CpuTuneDef.MAX_PERIOD; + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java index 0060e1d7ed4d..e6f63e852f85 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java @@ -115,6 +115,145 @@ public void testGpuDef_withComplexPciAddress() { assertTrue(gpuXml.contains("")); } + @Test + public void testGpuDef_withFullPciAddressDomainZero() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0000:00:02.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0001:65:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withNvidiaStyleEightDigitDomain() { + // nvidia-smi reports PCI addresses with an 8-digit domain (e.g. "00000001:af:00.1"). + // generatePciXml must normalize it to the canonical 4-digit form "0x0001". + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "00000001:af:00.1", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressVfNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo vfGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "VF-Profile", + "VF-Profile", + "0002:81:00.3", + "10de", + "NVIDIA Corporation", + "1eb8", + "Tesla T4" + ); + gpuDef.defGpu(vfGpuInfo); + + String gpuXml = gpuDef.toString(); + + // Non-passthrough NVIDIA VFs should be unmanaged + assertTrue(gpuXml.contains("")); + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withLegacyShortBdfDefaultsDomainToZero() { + // Backward compatibility: short BDF with no domain segment must still + // produce a valid libvirt address with domain 0x0000. + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "af:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withInvalidBusAddressThrows() { + String[] invalidAddresses = { + "notahex:00.0", // non-hex bus + "gg:00:02.0", // non-hex domain + "00:02:03:04", // too many colon-separated parts + "00", // missing slot/function + "00:02", // missing function (no dot) + "00:02.0.1", // extra dot in ss.f + }; + for (String addr : invalidAddresses) { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo info = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + addr, + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(info); + try { + String ignored = gpuDef.toString(); + fail("Expected IllegalArgumentException for address: " + addr + " but got: " + ignored); + } catch (IllegalArgumentException e) { + assertTrue("Exception message should contain the bad address", + e.getMessage().contains(addr)); + } + } + } + @Test public void testGpuDef_withNullDeviceType() { LibvirtGpuDef gpuDef = new LibvirtGpuDef(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java index 3cad9c27a687..74090723331a 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java @@ -52,6 +52,7 @@ public void setUp() { @Test public void testCheckInstanceCommand_success() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(false); Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(true); Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); assertTrue(answer.getResult()); @@ -59,9 +60,33 @@ public void testCheckInstanceCommand_success() { @Test public void testCheckInstanceCommand_failure() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(false); Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(false); Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); assertFalse(answer.getResult()); assertTrue(StringUtils.isNotBlank(answer.getDetails())); } + + @Test + public void testCheckInstanceCommand_vddkSuccess() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(true); + Mockito.when(checkConvertInstanceCommandMock.getVddkLibDir()).thenReturn("/opt/vmware-vddk/vmware-vix-disklib-distrib"); + Mockito.when(libvirtComputingResourceMock.hostSupportsVddk("/opt/vmware-vddk/vmware-vix-disklib-distrib")).thenReturn(true); + + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + + assertTrue(answer.getResult()); + } + + @Test + public void testCheckInstanceCommand_vddkFailure() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(true); + Mockito.when(checkConvertInstanceCommandMock.getVddkLibDir()).thenReturn("/opt/vmware-vddk/vmware-vix-disklib-distrib"); + Mockito.when(libvirtComputingResourceMock.hostSupportsVddk("/opt/vmware-vddk/vmware-vix-disklib-distrib")).thenReturn(false); + + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + + assertFalse(answer.getResult()); + assertTrue(StringUtils.isNotBlank(answer.getDetails())); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java new file mode 100644 index 000000000000..a06a9c50bc1c --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; +import org.libvirt.DomainInfo.DomainState; +import org.libvirt.LibvirtException; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.CheckVirtualMachineAnswer; +import com.cloud.agent.api.CheckVirtualMachineCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.vm.VirtualMachine.PowerState; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckVirtualMachineCommandWrapperTest { + + private static final String VM_NAME = "i-2-3-VM"; + + @Mock + private LibvirtComputingResource libvirtComputingResource; + @Mock + private LibvirtUtilitiesHelper libvirtUtilitiesHelper; + @Mock + private Connect conn; + @Mock + private Domain domain; + + private LibvirtCheckVirtualMachineCommandWrapper wrapper; + private CheckVirtualMachineCommand command; + + @Before + public void setUp() throws LibvirtException { + wrapper = new LibvirtCheckVirtualMachineCommandWrapper(); + command = new CheckVirtualMachineCommand(VM_NAME); + + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnectionByVmName(VM_NAME)).thenReturn(conn); + } + + @Test + public void testExecuteVmPoweredOnReturnsStateAndVncPort() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_RUNNING; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5900); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOn, answer.getState()); + assertEquals(Integer.valueOf(5900), answer.getVncPort()); + } + + @Test + public void testExecuteVmPausedReturnsPowerUnknown() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_PAUSED; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5901); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerUnknown, answer.getState()); + assertEquals(Integer.valueOf(5901), answer.getVncPort()); + } + + @Test + public void testExecuteVmPoweredOffReturnsStateWithNullVncPort() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOff); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOff, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteVmStateUnknownReturnsStateWithNullVncPort() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerUnknown); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerUnknown, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteVmPoweredOnWithNullVncPort() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_RUNNING; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(null); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOn, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteLibvirtExceptionOnGetConnectionReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("Connection refused"); + when(libvirtUtilitiesHelper.getConnectionByVmName(VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("Connection refused", answer.getDetails()); + } + + @Test + public void testExecuteLibvirtExceptionOnGetVncPortReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("VNC port error"); + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("VNC port error", answer.getDetails()); + } + + @Test + public void testExecuteLibvirtExceptionOnDomainLookupReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("Domain not found"); + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5900); + when(conn.domainLookupByName(VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("Domain not found", answer.getDetails()); + } + + @Test + public void testExecuteCallsGetLibvirtUtilitiesHelper() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOff); + + wrapper.execute(command, libvirtComputingResource); + + verify(libvirtComputingResource).getLibvirtUtilitiesHelper(); + verify(libvirtUtilitiesHelper).getConnectionByVmName(VM_NAME); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index 4d55ac2bc73a..c30fa2f49482 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -18,6 +18,7 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import java.nio.file.Files; import java.util.List; import java.util.UUID; @@ -189,4 +190,127 @@ public void testAddExtraParamsToScriptDifferentArgs() { Mockito.verify(script).add("-x"); Mockito.verify(script).add("-v"); } + + @Test + public void testPerformInstanceConversionUsingVddkUsesConfiguredLibguestfsBackend() { + RemoteInstanceTO remoteInstanceTO = Mockito.mock(RemoteInstanceTO.class); + Mockito.when(remoteInstanceTO.getVcenterHost()).thenReturn("vcenter.local"); + Mockito.when(remoteInstanceTO.getVcenterUsername()).thenReturn("administrator@vsphere.local"); + Mockito.when(remoteInstanceTO.getVcenterPassword()).thenReturn("secret"); + Mockito.when(remoteInstanceTO.getDatacenterName()).thenReturn("dc1"); + Mockito.when(remoteInstanceTO.getClusterName()).thenReturn("cluster1"); + Mockito.when(remoteInstanceTO.getHostName()).thenReturn("host1"); + Mockito.doReturn("28:19:A6:1C:90:ED:46:D7:1C:86:BC:F6:13:52:F0:B9:19:81:0D:81") + .when(convertInstanceCommandWrapper).getVcenterThumbprint(Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()); + + try (MockedStatic filesMock = Mockito.mockStatic(Files.class); + MockedConstruction