From 5055250f1b4a436ccbca25cb2bf36d9a1588e75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Busso?= <90727999+agustinbusso@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:28:23 -0300 Subject: [PATCH 01/33] Update deprecated model for translations --- .../Handlers/LanguageTranslationHandler.php | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php b/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php index fda05562ff..e8718fe55a 100644 --- a/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php +++ b/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php @@ -18,8 +18,8 @@ public function __construct() { parent::__construct(); $this->config = [ - 'model' => 'text-davinci-003', - 'max_tokens' => 2200, + 'model' => 'gpt-3.5-turbo-16k', + 'max_tokens' => 6000, 'temperature' => 0, 'top_p' => 1, 'n' => 1, @@ -52,7 +52,10 @@ public function generatePrompt(String $type = null, String $json_list) : Object $prompt = $this->replaceLanguage($prompt, $this->targetLanguage['humanLanguage']); $prompt = $this->replaceStopSequence($prompt); $this->config['prompt'] = $prompt; - + $this->config['messages'] = [[ + 'role' => 'user', + 'content' => $prompt, + ]]; return $this; } @@ -61,17 +64,25 @@ public function execute() $listCharCount = strlen($this->json_list); $totalChars = $listCharCount * 3; $currentChunkCount = 0; + $config = $this->getConfig(); + unset($config['prompt']); $client = app(Client::class); $stream = $client - ->completions() - ->createStreamed(array_merge($this->getConfig())); + ->chat() + ->createStreamed(array_merge($config)); $fullResponse = ''; foreach ($stream as $response) { - $currentChunkCount += strlen($response->choices[0]->text); - self::sendResponse($response->choices[0]->text, $currentChunkCount, $totalChars); - $fullResponse .= $response->choices[0]->text; + if (array_key_exists('content', $response->choices[0]->toArray()['delta'])) { + $currentChunkCount += strlen($response->choices[0]->toArray()['delta']['content']); + self::sendResponse( + $response->choices[0]->toArray()['delta']['content'], + $currentChunkCount, + $totalChars + ); + $fullResponse .= $response->choices[0]->toArray()['delta']['content']; + } } return $this->formatResponse($fullResponse); From 57a3c5c3d1a0a5bc717463864accc24bbf63e1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Busso?= <90727999+agustinbusso@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:28:32 -0300 Subject: [PATCH 02/33] Fix format --- ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php | 1 + 1 file changed, 1 insertion(+) diff --git a/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php b/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php index e8718fe55a..b8a0571fa0 100644 --- a/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php +++ b/ProcessMaker/Ai/Handlers/LanguageTranslationHandler.php @@ -56,6 +56,7 @@ public function generatePrompt(String $type = null, String $json_list) : Object 'role' => 'user', 'content' => $prompt, ]]; + return $this; } From ac68b1d18ac566912b6ae1d62dfb08ef20f5011b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Busso?= <90727999+agustinbusso@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:57:04 -0300 Subject: [PATCH 03/33] Update NlqToCategory to use gtp-3.5 model instead deprecated davinci model --- .../Ai/Handlers/NlqToCategoryHandler.php | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php b/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php index c89c17d1dd..4f7020e154 100644 --- a/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php +++ b/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php @@ -10,7 +10,7 @@ public function __construct() { parent::__construct(); $this->config = [ - 'model' => 'text-davinci-003', + 'model' => 'gpt-3.5-turbo', 'max_tokens' => 20, 'temperature' => 0, 'top_p' => 1, @@ -34,7 +34,10 @@ public function generatePrompt(String $type = null, String $question) : Object $prompt = $this->replaceStopSequence($prompt); $prompt = $this->replaceDefaultType($prompt, $type); - $this->config['prompt'] = $prompt; + $this->config['messages'] = [[ + 'role' => 'user', + 'content' => $prompt, + ]]; return $this; } @@ -43,15 +46,17 @@ public function execute() { $client = app(Client::class); $response = $client - ->completions() - ->create(array_merge($this->getConfig())); - + ->chat() + ->create( + array_merge($this->getConfig() + ) + ); return $this->formatResponse($response); } private function formatResponse($response) { - $result = ltrim($response->choices[0]->text); + $result = ltrim($response->choices[0]->message->content); return [strtolower($result), $response->usage, $this->question]; } From 88f28aeb07d2e4f51ac5866ed661d41e1e645510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Busso?= <90727999+agustinbusso@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:57:16 -0300 Subject: [PATCH 04/33] Fix format --- ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php | 1 + 1 file changed, 1 insertion(+) diff --git a/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php b/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php index 4f7020e154..54130e07fe 100644 --- a/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php +++ b/ProcessMaker/Ai/Handlers/NlqToCategoryHandler.php @@ -51,6 +51,7 @@ public function execute() array_merge($this->getConfig() ) ); + return $this->formatResponse($response); } From 4b58ce36035341f18a22111e4052aec1405154e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Busso?= <90727999+agustinbusso@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:57:33 -0300 Subject: [PATCH 05/33] Update NlqToPmql to use gtp-3.5 model instead deprecated davinci model --- ProcessMaker/Ai/Handlers/NlqToPmqlHandler.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ProcessMaker/Ai/Handlers/NlqToPmqlHandler.php b/ProcessMaker/Ai/Handlers/NlqToPmqlHandler.php index 81d43b4dc3..c2eae050f4 100644 --- a/ProcessMaker/Ai/Handlers/NlqToPmqlHandler.php +++ b/ProcessMaker/Ai/Handlers/NlqToPmqlHandler.php @@ -10,7 +10,7 @@ public function __construct() { parent::__construct(); $this->config = [ - 'model' => 'text-davinci-003', + 'model' => 'gpt-3.5-turbo', 'max_tokens' => 1900, 'temperature' => 0, 'top_p' => 1, @@ -34,7 +34,10 @@ public function generatePrompt(String $type = null, String $question) : Object $prompt = $this->replaceStopSequence($prompt); $prompt = $this->replaceWithCurrentYear($prompt); - $this->config['prompt'] = $prompt; + $this->config['messages'] = [[ + 'role' => 'user', + 'content' => $prompt, + ]]; return $this; } @@ -43,15 +46,18 @@ public function execute() { $client = app(Client::class); $response = $client - ->completions() - ->create(array_merge($this->getConfig())); + ->chat() + ->create( + array_merge($this->getConfig() + ) + ); return $this->formatResponse($response); } private function formatResponse($response) { - $result = ltrim($response->choices[0]->text); + $result = ltrim($response->choices[0]->message->content); $result = explode('Question:', $result)[0]; $result = rtrim(rtrim(str_replace("\n", '', $result))); $result = str_replace('\'', '', $result); From 17b12fbeaa917b0c722fb3e70904b755f44b7848 Mon Sep 17 00:00:00 2001 From: David Callizaya Date: Wed, 20 Dec 2023 10:35:22 -0400 Subject: [PATCH 06/33] Add custom package-zj-adps --- composer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/composer.json b/composer.json index f7f9c9a28f..d0656f2a59 100644 --- a/composer.json +++ b/composer.json @@ -112,6 +112,7 @@ "package-csi": "dev-fall", "package-jxchange": "dev-fall", "package-zjukumari-logs": "dev-fall", + "package-zj-adps": "dev-fall", "package-zj-wingate": "dev-fall", "package-banner-sync": "dev-fall", "package-service-task": "dev-fall", From e045e6f9fadc2b138ed48118f513032ddeeb67dc Mon Sep 17 00:00:00 2001 From: Miguel Angel Date: Wed, 20 Dec 2023 14:35:21 -0400 Subject: [PATCH 07/33] fix: logic for task redirection --- ProcessMaker/Http/Controllers/RequestController.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ProcessMaker/Http/Controllers/RequestController.php b/ProcessMaker/Http/Controllers/RequestController.php index 8b0ffb0b8a..a4add102b4 100644 --- a/ProcessMaker/Http/Controllers/RequestController.php +++ b/ProcessMaker/Http/Controllers/RequestController.php @@ -89,8 +89,15 @@ public function show(ProcessRequest $request, Media $mediaItems) ->where('status', 'ACTIVE') ->orderBy('id')->first(); + // If the interstitial is enabled on the start event, then use it as the task + if ($active) { + $task = $allowInterstitial ? $startEvent : $active; + } else { + $task = $startEvent; + } + return redirect(route('tasks.edit', [ - 'task' => $active ? $active->getKey() : $startEvent->getKey() + 'task' => $task->getKey(), ])); } } From 28ce192b0482e7f88db50d9b0cb8d0f8d3618777 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Wed, 20 Dec 2023 21:41:39 -0800 Subject: [PATCH 08/33] Update enterprise package --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index d0656f2a59..7b7574c18e 100644 --- a/composer.json +++ b/composer.json @@ -136,7 +136,7 @@ "docker-executor-node-ssr": "1.5.0", "package-actions-by-email": "1.15.0", "package-advanced-user-manager": "1.8.0", - "package-ai": "1.3.0", + "package-ai": "1.3.1", "package-analytics-reporting": "1.3.2", "package-auth": "1.15.3", "package-cdata": "1.0.2", From a84a6a72cf1f4a58872b77bc03ad6e7d952dedec Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Wed, 20 Dec 2023 21:42:52 -0800 Subject: [PATCH 09/33] Version 4.8.3+nightly-20231220 Build #08a2860b --- composer.json | 6 +++--- composer.lock | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/composer.json b/composer.json index 7b7574c18e..7eb4cbbdaf 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "processmaker/processmaker", - "version": "4.8.2", + "version": "4.8.3+nightly-20231220", "description": "BPM PHP Software", "keywords": [ "php bpm processmaker" @@ -96,7 +96,7 @@ "Gmail" ], "processmaker": { - "build": "6ddfe95a", + "build": "08a2860b", "custom": { "package-ellucian-ethos": "1.14.2", "package-plaid": "1.3.1", @@ -172,7 +172,7 @@ "microservices": { "pmai": "fall-2023" }, - "release": "Fall 2023" + "release": "Fall 2023 Pre-Release" } }, "scripts": { diff --git a/composer.lock b/composer.lock index 5ff1a4de87..43f672836a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0e2bf759df812117e65fe62678170f76", + "content-hash": "2046403b857747790bd80ddee43fa577", "packages": [ { "name": "aws/aws-crt-php", diff --git a/package-lock.json b/package-lock.json index 3236caecd2..01bd344910 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@processmaker/processmaker", - "version": "4.8.2", + "version": "4.8.3+nightly-20231220", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@processmaker/processmaker", - "version": "4.8.2", + "version": "4.8.3+nightly-20231220", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/package.json b/package.json index 512a19b07b..e6dbc62cdb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@processmaker/processmaker", - "version": "4.8.2", + "version": "4.8.3+nightly-20231220", "description": "ProcessMaker 4", "author": "DevOps ", "license": "ISC", From 1c396d1fdfb9de10e6cb526c4888687973d36773 Mon Sep 17 00:00:00 2001 From: Alex Runyan Date: Thu, 21 Dec 2023 14:47:32 -0500 Subject: [PATCH 10/33] Added migration to convert status column from enum to varchar --- ...104245_check_fix_status_at_users_table.php | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 database/migrations/2023_12_18_104245_check_fix_status_at_users_table.php diff --git a/database/migrations/2023_12_18_104245_check_fix_status_at_users_table.php b/database/migrations/2023_12_18_104245_check_fix_status_at_users_table.php new file mode 100644 index 0000000000..da37de0ed6 --- /dev/null +++ b/database/migrations/2023_12_18_104245_check_fix_status_at_users_table.php @@ -0,0 +1,32 @@ +Type, 0, 4) === 'enum'; + if (!$isEnum) { + return; + } + // change the column status to varchar + DB::statement('ALTER TABLE users MODIFY COLUMN status VARCHAR(255) NOT NULL DEFAULT "ACTIVE"'); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + } +}; From f7b97fae767fcf6196756b442fda617da3151f85 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Thu, 21 Dec 2023 18:23:34 -0800 Subject: [PATCH 11/33] Version 4.8.3+nightly-20231221 Build #4b4e1f2d --- composer.json | 4 ++-- composer.lock | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.json b/composer.json index 7eb4cbbdaf..21c0f71375 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "processmaker/processmaker", - "version": "4.8.3+nightly-20231220", + "version": "4.8.3+nightly-20231221", "description": "BPM PHP Software", "keywords": [ "php bpm processmaker" @@ -96,7 +96,7 @@ "Gmail" ], "processmaker": { - "build": "08a2860b", + "build": "4b4e1f2d", "custom": { "package-ellucian-ethos": "1.14.2", "package-plaid": "1.3.1", diff --git a/composer.lock b/composer.lock index 43f672836a..43541a65a6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2046403b857747790bd80ddee43fa577", + "content-hash": "d4dfab19f92b59f3b60bc39fa568a11f", "packages": [ { "name": "aws/aws-crt-php", diff --git a/package-lock.json b/package-lock.json index 01bd344910..755025cda0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@processmaker/processmaker", - "version": "4.8.3+nightly-20231220", + "version": "4.8.3+nightly-20231221", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@processmaker/processmaker", - "version": "4.8.3+nightly-20231220", + "version": "4.8.3+nightly-20231221", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/package.json b/package.json index e6dbc62cdb..d1109a053e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@processmaker/processmaker", - "version": "4.8.3+nightly-20231220", + "version": "4.8.3+nightly-20231221", "description": "ProcessMaker 4", "author": "DevOps ", "license": "ISC", From ea182383574ab6f976346ba9a931193b4f2d6a47 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Fri, 22 Dec 2023 13:55:50 -0800 Subject: [PATCH 12/33] Version 4.8.3 Build #ad1d0941 --- composer.json | 6 +++--- composer.lock | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/composer.json b/composer.json index 21c0f71375..0d27b559e1 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "processmaker/processmaker", - "version": "4.8.3+nightly-20231221", + "version": "4.8.3", "description": "BPM PHP Software", "keywords": [ "php bpm processmaker" @@ -96,7 +96,7 @@ "Gmail" ], "processmaker": { - "build": "4b4e1f2d", + "build": "ad1d0941", "custom": { "package-ellucian-ethos": "1.14.2", "package-plaid": "1.3.1", @@ -172,7 +172,7 @@ "microservices": { "pmai": "fall-2023" }, - "release": "Fall 2023 Pre-Release" + "release": "Fall 2023" } }, "scripts": { diff --git a/composer.lock b/composer.lock index 43541a65a6..2abfa40398 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d4dfab19f92b59f3b60bc39fa568a11f", + "content-hash": "5159980e43b36cd56a051795b1389dba", "packages": [ { "name": "aws/aws-crt-php", diff --git a/package-lock.json b/package-lock.json index 755025cda0..38b7c72aec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@processmaker/processmaker", - "version": "4.8.3+nightly-20231221", + "version": "4.8.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@processmaker/processmaker", - "version": "4.8.3+nightly-20231221", + "version": "4.8.3", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/package.json b/package.json index d1109a053e..06113ae944 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@processmaker/processmaker", - "version": "4.8.3+nightly-20231221", + "version": "4.8.3", "description": "ProcessMaker 4", "author": "DevOps ", "license": "ISC", From 02991687da8100dafcbf4416e772eb92c65d17b8 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Mon, 8 Jan 2024 14:38:07 -0800 Subject: [PATCH 13/33] Switch to k8s CICD --- .github/workflows/deploy-pm4.yml | 295 ++++++++++++------------------- 1 file changed, 111 insertions(+), 184 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 0ce642d97a..6ed37d0142 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -1,212 +1,139 @@ -name: BUILD-PM4 +name: deploy-k8s +run-name: ${{ github.actor }} send deploy EKS 🚀 on: - #push: - # branches: - # - kr-github-actions pull_request: types: [opened, reopened, synchronize, edited, closed] - schedule: - - cron: '30 2 * * *' # every day at midnight - #workflow_dispatch: - #pull_request: - # branches: - # - main + #schedule: + # - cron: '30 2 * * *' # every day at midnight workflow_dispatch: workflow_call: env: - SHA: ${{github.event.pull_request.head.sha}} - PROJECT: ${{github.event.pull_request.head.repo.name}} - CI_PR_BODY: ${{ github.event_name == 'schedule' && 'ci:deploy' || github.event.pull_request.body }} - PACKAGE_URL: ${{github.event.pull_request.head.repo.ssh_url}} - PACKAGE_BRANCH: ${{github.event.pull_request.head.ref}} - #MY_GITHUB_TOKEN: ${{ secrets.GH_STATUS_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GIT_TOKEN }} - #GIT_TOKEN: ${{ secrets.MY_GH_TOKEN }} - OWNER: ${{ github.event.pull_request.head.repo.owner.login }} - #Other Parameters aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} - aws-url: ${{ secrets.AWS_URL }} - STM_TOKEN: ${{ secrets.STM_TOKEN }} - CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'develop' }} - CI_PROJECT: ${{github.event.pull_request.head.repo.name || 'processmaker' }} - CI_TEST: $CI_PROJECT - IMAGE_TAG1: $(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g") - GITHUB_COMMENT: ${{ secrets.GH_COMMENT }} + AWS_URL: ${{ secrets.AWS_URL }} pull_req_id: ${{github.event.pull_request.number}} - BASE: ${{ contains(github.event.pull_request.body, 'ci:php81') && 'ci-base' || 'ci-base-php82' }} - CDATA_LICENSE_DOCUSIGN: ${{ secrets.CDATA_LICENSE_DOCUSIGN }} - CDATA_LICENSE_EXCEL: ${{ secrets.CDATA_LICENSE_EXCEL }} - CDATA_LICENSE_GITHUB: ${{ secrets.CDATA_LICENSE_GITHUB }} - CDATA_LICENSE_SLACK: ${{ secrets.CDATA_LICENSE_SLACK }} + DATE: $(date -d '-1 day' '+%Y-%m-%d'|sed 's/-//g') + CURRENT_DATE: $(date '+%Y-%m-%d %H:%M:%S'|sed 's/-//g') + CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'next' }} + CI_PROJECT: ${{github.event.pull_request.head.repo.name || 'processmaker' }} + CI_PR_BODY: ${{ github.event_name == 'schedule' && 'No ci tags needed here' || github.event.pull_request.body }} + IMAGE_TAG: $(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g") + DEPLOY: ${{ secrets.DEPLOY }} + GH_USER: ${{ secrets.GH_USER }} + GH_EMAIL: ${{ secrets.GH_EMAIL }} + GITHUB_COMMENT: ${{ secrets.GH_COMMENT }} + DOM_EKS: ${{ secrets.DOM_EKS }} + GITHUB_TOKEN: ${{ secrets.GIT_TOKEN }} + BUILD_BASE: ${{ (contains(github.event.pull_request.body, 'ci:build-base') || github.event_name == 'schedule') && '1' || '0' }} + BASE_IMAGE: ${{ secrets.REGISTRY_HOST }}/processmaker/processmaker:base + K8S_BRANCH: develop concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - job1: - name: build-stm-image + imageEKS: + name: build-docker-image-EKS if: github.event.action != 'closed' - runs-on: ${{ vars.RUNNER }} - steps: - - name: Export Params + runs-on: ${{ vars.RUNNER }} + steps: + - name: Set image name run: | - echo "Env Check: CI_PROJECT: $CI_PROJECT CI_PACKAGE_BRANCH: $CI_PACKAGE_BRANCH CI_PR_BODY: $CI_PR_BODY BASE: $BASE" - echo "REPOSITORY=${{env.aws-url}}/enterprise" >> $GITHUB_ENV - echo "TAG=${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "IMAGE=${{env.aws-url}}/enterprise:${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - #Additional - echo "CACHEBUSTER="$(date +%s) >> $GITHUB_ENV - - name: Clone Repo STM + RESOLVED_IMAGE_TAG=${{ env.IMAGE_TAG }} + echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:$RESOLVED_IMAGE_TAG" >> $GITHUB_ENV + - name: Clone repo K8S run: | - git clone --depth 1 -b cicd "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-stm-docker.git" pm4-stm-docker - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ env.aws-access-key-id }} - aws-secret-access-key: ${{ env.aws-secret-access-key }} - aws-region: ${{ env.aws-region }} - - name: Login to ECR - run: | - aws ecr get-login-password | docker login --username AWS --password-stdin ${{env.aws-url}} - - name: Build and Push the base images - if: contains(github.event.pull_request.body, 'ci:build-base') || github.event_name == 'schedule' + # TODO: Change branch when pm4 k8s distribution is released + echo "IMAGE: ${{ env.IMAGE }}" + git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + - name: Generate image EKS run: | - cd pm4-stm-docker - docker-compose build --no-cache base-php82 - docker-compose build --no-cache cache - docker push ${REPOSITORY}:ci-base-php82 - docker push ${REPOSITORY}:ci-cache - - name: Build and Push the image to ECR + cd pm4-k8s-distribution/images + branch=$CI_PACKAGE_BRANCH tag=${{env.IMAGE_TAG}} bash build.k8s-cicd.sh + echo "VERSION=${{ env.IMAGE_TAG }}" >> $GITHUB_ENV + - name: List Images run: | - cd pm4-stm-docker - docker-compose build processmaker - docker push ${IMAGE} - job2: - name: deploy-stm - if: github.event.action != 'closed' - needs: job1 - runs-on: ${{ vars.RUNNER }} - container: - image: cimg/php:7.4 - options: --user root - steps: - - name: Export Params + docker images + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: processmaker/enterprise:${{ env.VERSION }} + format: 'table' + exit-code: '0' + ignore-unfixed: false + vuln-type: 'os,library' + scanners: 'vuln,secret' + severity: 'MEDIUM,HIGH,CRITICAL' + env: + TRIVY_TIMEOUT: 30m + - name: Login to Harbor + uses: docker/login-action@v2 + with: + registry: ${{ secrets.REGISTRY_HOST }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Push Enterprise Image to Harbor + run: | + docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + deployEKS: + name: build-deploy-EKS + if: contains(github.event.pull_request.body, 'ci:deploy') + needs: imageEKS + runs-on: self-hosted + steps: + - name: Clone private repository run: | - echo "Env Check: CI_PROJECT: $CI_PROJECT CI_PACKAGE_BRANCH: $CI_PACKAGE_BRANCH CI_PR_BODY: $CI_PR_BODY" - echo "REPOSITORY=${{env.aws-url}}/enterprise" >> $GITHUB_ENV - echo "TAG=${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "IMAGE_TAG=${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "IMAGE=${{env.aws-url}}/enterprise:${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "STM_TOKEN=${{env.STM_TOKEN}}" >> $GITHUB_ENV - - name: Clone Repo STM + git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd + - name: CreateDB + run: | + cd argocd + deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml + kubectl get namespace ci-processmaker-ns-pm4 + namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') + kubectl apply -f template-db.yaml + - name: Install pm4-tools run: | - git clone --depth 1 -b cicd "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-stm-docker.git" pm4-stm-docker - cd pm4-stm-docker - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ env.aws-access-key-id }} - aws-secret-access-key: ${{ env.aws-secret-access-key }} - aws-region: ${{ env.aws-region }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Deploy STM - id: stm + git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + cd pm4-k8s-distribution/images/pm4-tools + composer install --no-interaction + cd .. + - name: Deploy instance EKS run: | - mkdir -p /tmp/workspace - cd pm4-stm-docker/deploy-stm - composer install --no-dev - php run.php - if [ -f "url.txt" ]; then - INSTANCE_URL=$(cat url.txt) + cd argocd + deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + current_datetime=$(echo -n ${{env.CURRENT_DATE}} | md5sum | head -c 10) + echo "NAMESPACE : ci-$deploy-ns-pm4" + helm repo add processmaker ${{ secrets.HELM_REPO }} --username ${{ secrets.HELM_USERNAME }} --password ${{ secrets.HELM_PASSWORD }} && helm repo update + if ! kubectl get namespace/ci-$deploy-ns-pm4 ; then + echo "Creating Deploy :: $deploy" + sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml + sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml + cat template-instance.yaml + helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 + else + echo "Bouncing Instance "; + sed -i "s/{{instance}}/ci-$deploy/g" template-bounce.yaml + sed -i "s/{{current_datetime}}/$current_datetime/g" template-bounce.yaml + helm upgrade --timeout 20m ci-$deploy processmaker/enterprise --version 2.1.0 + kubectl apply -f template-bounce.yaml fi - echo "Instance URL: '${INSTANCE_URL}'" + export INSTANCE_URL=https://ci-$deploy$DOM_EKS echo "INSTANCE_URL=${INSTANCE_URL}" >> "$GITHUB_ENV" - - name: Publish the URL to the Github PR - if: success() || steps.stm.conclusion == 'success' + ../pm4-k8s-distribution/images/pm4-tools/pm wait-for-instance-ready + - name: Comment Instance run: | - cd pm4-stm-docker echo "Instance URL: '${INSTANCE_URL}'" - bash ./github_comment.sh "$PROJECT" "$pull_req_id" - job3: - name: run-phpunit - if: github.event.action != 'closed' - needs: job1 - runs-on: ${{ vars.RUNNER }} - steps: - - name: Export Params - run: | - echo "REPOSITORY=${{env.aws-url}}/enterprise" >> $GITHUB_ENV - echo "TAG=${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "IMAGE=${{env.aws-url}}/enterprise:${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - - name: Clone Repo STM - run: | - git clone --depth 1 -b cicd "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-stm-docker.git" pm4-stm-docker - cd pm4-stm-docker - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ env.aws-access-key-id }} - aws-secret-access-key: ${{ env.aws-secret-access-key }} - aws-region: ${{ env.aws-region }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: PHPUnits - run: | - cd pm4-stm-docker - docker pull $IMAGE - docker-compose down -v - docker-compose build phpunit - docker-compose run phpunit - #job4: - # name: run-benchmarks - # needs: job2 - # runs-on: ${{ vars.RUNNER }} - # steps: - # - name: my-step - # run: | - # echo "Fifth Step" - #job5: - # name: run-cypress - # needs: job2 - # runs-on: ${{ vars.RUNNER }} - # steps: - # - name: my-step - # run: | - # echo "fourth Step" - #job6: - # name: run-cypress-qa - # needs: job2 - # runs-on: ${{ vars.RUNNER }} - # steps: - # - name: my-step - # run: | - # echo "Sixt Step" - job7: + bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" + deleteEKS: name: Delete Instance if: github.event.action == 'closed' - runs-on: ${{ vars.RUNNER }} - container: - image: cimg/php:7.4 - options: --user root - steps: - - name: Export Params - run: | - echo "REPOSITORY=${{env.aws-url}}/enterprise" >> $GITHUB_ENV - echo "TAG=${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "IMAGE_TAG=${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "IMAGE=${{env.aws-url}}/enterprise:${{env.IMAGE_TAG1}}" >> $GITHUB_ENV - echo "STM_TOKEN=${{env.STM_TOKEN}}" >> $GITHUB_ENV - - name: Clone Repo STM - run: | - git clone --depth 1 -b cicd "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-stm-docker.git" pm4-stm-docker - cd pm4-stm-docker - - name: Delete Instance STM - run: | - mkdir -p /tmp/workspace - cd pm4-stm-docker/deploy-stm - composer install --no-dev - php run-delete-instance.php + runs-on: self-hosted + steps: + - name: Delete instance EKS + run: | + deploy=$(echo -n $IMAGE_TAG | md5sum | head -c 10) + echo "Deleting Instace :: ci-$deploy" + helm delete ci-$deploy + kubectl delete namespace ci-$deploy-ns-pm4 From 93b3d1bc9740e6b9aa4f19d1445c217daa736566 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Wed, 10 Jan 2024 17:03:55 -0800 Subject: [PATCH 14/33] Enable phpunit --- .github/workflows/deploy-pm4.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 6ed37d0142..edd290853a 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -137,3 +137,33 @@ jobs: echo "Deleting Instace :: ci-$deploy" helm delete ci-$deploy kubectl delete namespace ci-$deploy-ns-pm4 + + runPhpUnit: + name: run-phpunit + if: github.event.action != 'closed' + needs: imageEKS + runs-on: ${{ vars.RUNNER }} + steps: + - name: Export Params + run: | + echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}}" >> $GITHUB_ENV + - name: Clone Repo STM + run: | + git clone --depth 1 -b cicd "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-stm-docker.git" pm4-stm-docker + cd pm4-stm-docker + - name: Login to Harbor + uses: docker/login-action@v2 + with: + registry: ${{ secrets.REGISTRY_HOST }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: PHPUnits + run: | + cd pm4-stm-docker + docker pull $IMAGE + docker-compose down -v + docker-compose build phpunit + docker-compose run phpunit From b5aaa3f4853c88966c30e620e318d7dddf028ad6 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Wed, 10 Jan 2024 17:32:01 -0800 Subject: [PATCH 15/33] Update deploy-pm4.yml --- .github/workflows/deploy-pm4.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index edd290853a..61ac8d4a44 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -157,9 +157,6 @@ jobs: registry: ${{ secrets.REGISTRY_HOST }} username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - name: PHPUnits run: | cd pm4-stm-docker From 9ed12ba07e233fb8a7421405725c4f800b1cda95 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Thu, 11 Jan 2024 08:10:59 -0800 Subject: [PATCH 16/33] Enable cicd phpunit --- .github/workflows/deploy-pm4.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 61ac8d4a44..a48f2443cc 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -27,7 +27,7 @@ env: GITHUB_TOKEN: ${{ secrets.GIT_TOKEN }} BUILD_BASE: ${{ (contains(github.event.pull_request.body, 'ci:build-base') || github.event_name == 'schedule') && '1' || '0' }} BASE_IMAGE: ${{ secrets.REGISTRY_HOST }}/processmaker/processmaker:base - K8S_BRANCH: develop + K8S_BRANCH: cicd-enable-phpunit concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -43,7 +43,6 @@ jobs: echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:$RESOLVED_IMAGE_TAG" >> $GITHUB_ENV - name: Clone repo K8S run: | - # TODO: Change branch when pm4 k8s distribution is released echo "IMAGE: ${{ env.IMAGE }}" git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - name: Generate image EKS @@ -147,10 +146,10 @@ jobs: - name: Export Params run: | echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}}" >> $GITHUB_ENV - - name: Clone Repo STM + - name: Clone repo K8S run: | - git clone --depth 1 -b cicd "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-stm-docker.git" pm4-stm-docker - cd pm4-stm-docker + echo "IMAGE: ${{ env.IMAGE }}" + git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - name: Login to Harbor uses: docker/login-action@v2 with: @@ -159,8 +158,8 @@ jobs: password: ${{ secrets.REGISTRY_PASSWORD }} - name: PHPUnits run: | - cd pm4-stm-docker - docker pull $IMAGE - docker-compose down -v - docker-compose build phpunit + cd pm4-stm-docker + docker pull $IMAGE + docker-compose down -v + docker-compose build phpunit docker-compose run phpunit From 149239af6b0918adb7e497604854475712159ba4 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Thu, 11 Jan 2024 08:33:29 -0800 Subject: [PATCH 17/33] Fix folder location --- .github/workflows/deploy-pm4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index a48f2443cc..0d6b8e0633 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -158,7 +158,7 @@ jobs: password: ${{ secrets.REGISTRY_PASSWORD }} - name: PHPUnits run: | - cd pm4-stm-docker + cd pm4-k8s-distribution/images/pm4-tools docker pull $IMAGE docker-compose down -v docker-compose build phpunit From 63b0a7417de2a425c3b6a032d9c02158eeb8d268 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Thu, 11 Jan 2024 11:51:01 -0800 Subject: [PATCH 18/33] Set branch back to develop now that its merged --- .github/workflows/deploy-pm4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 0d6b8e0633..120686f754 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -27,7 +27,7 @@ env: GITHUB_TOKEN: ${{ secrets.GIT_TOKEN }} BUILD_BASE: ${{ (contains(github.event.pull_request.body, 'ci:build-base') || github.event_name == 'schedule') && '1' || '0' }} BASE_IMAGE: ${{ secrets.REGISTRY_HOST }}/processmaker/processmaker:base - K8S_BRANCH: cicd-enable-phpunit + K8S_BRANCH: develop concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true From 22beab86d9e68d8fe3f96e5bb536033fe929ee5a Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 21:46:12 -0400 Subject: [PATCH 19/33] Add Second alternative --- .github/workflows/deploy-pm4.yml | 288 +++++++++++++++++-------------- 1 file changed, 159 insertions(+), 129 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 120686f754..c115d6383e 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -5,6 +5,9 @@ on: types: [opened, reopened, synchronize, edited, closed] #schedule: # - cron: '30 2 * * *' # every day at midnight + push: + branches: + - FOUR-13153 workflow_dispatch: workflow_call: env: @@ -15,7 +18,7 @@ env: pull_req_id: ${{github.event.pull_request.number}} DATE: $(date -d '-1 day' '+%Y-%m-%d'|sed 's/-//g') CURRENT_DATE: $(date '+%Y-%m-%d %H:%M:%S'|sed 's/-//g') - CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'next' }} + CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'FOUR-13151' }} CI_PROJECT: ${{github.event.pull_request.head.repo.name || 'processmaker' }} CI_PR_BODY: ${{ github.event_name == 'schedule' && 'No ci tags needed here' || github.event.pull_request.body }} IMAGE_TAG: $(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g") @@ -32,134 +35,161 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - imageEKS: - name: build-docker-image-EKS - if: github.event.action != 'closed' + deploy: runs-on: ${{ vars.RUNNER }} + steps: - - name: Set image name - run: | - RESOLVED_IMAGE_TAG=${{ env.IMAGE_TAG }} - echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:$RESOLVED_IMAGE_TAG" >> $GITHUB_ENV - - name: Clone repo K8S - run: | - echo "IMAGE: ${{ env.IMAGE }}" - git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - - name: Generate image EKS - run: | - cd pm4-k8s-distribution/images - branch=$CI_PACKAGE_BRANCH tag=${{env.IMAGE_TAG}} bash build.k8s-cicd.sh - echo "VERSION=${{ env.IMAGE_TAG }}" >> $GITHUB_ENV - - name: List Images - run: | - docker images - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: processmaker/enterprise:${{ env.VERSION }} - format: 'table' - exit-code: '0' - ignore-unfixed: false - vuln-type: 'os,library' - scanners: 'vuln,secret' - severity: 'MEDIUM,HIGH,CRITICAL' - env: - TRIVY_TIMEOUT: 30m - - name: Login to Harbor - uses: docker/login-action@v2 - with: - registry: ${{ secrets.REGISTRY_HOST }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_PASSWORD }} - - name: Push Enterprise Image to Harbor - run: | - docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} - docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} - deployEKS: - name: build-deploy-EKS - if: contains(github.event.pull_request.body, 'ci:deploy') - needs: imageEKS - runs-on: self-hosted - steps: - - name: Clone private repository - run: | - git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd - - name: CreateDB - run: | - cd argocd - deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) - sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml - kubectl get namespace ci-processmaker-ns-pm4 - namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') - kubectl apply -f template-db.yaml - - name: Install pm4-tools - run: | - git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - cd pm4-k8s-distribution/images/pm4-tools - composer install --no-interaction - cd .. - - name: Deploy instance EKS - run: | - cd argocd - deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) - current_datetime=$(echo -n ${{env.CURRENT_DATE}} | md5sum | head -c 10) - echo "NAMESPACE : ci-$deploy-ns-pm4" - helm repo add processmaker ${{ secrets.HELM_REPO }} --username ${{ secrets.HELM_USERNAME }} --password ${{ secrets.HELM_PASSWORD }} && helm repo update - if ! kubectl get namespace/ci-$deploy-ns-pm4 ; then - echo "Creating Deploy :: $deploy" - sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml - sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml - cat template-instance.yaml - helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 - else - echo "Bouncing Instance "; - sed -i "s/{{instance}}/ci-$deploy/g" template-bounce.yaml - sed -i "s/{{current_datetime}}/$current_datetime/g" template-bounce.yaml - helm upgrade --timeout 20m ci-$deploy processmaker/enterprise --version 2.1.0 - kubectl apply -f template-bounce.yaml - fi - export INSTANCE_URL=https://ci-$deploy$DOM_EKS - echo "INSTANCE_URL=${INSTANCE_URL}" >> "$GITHUB_ENV" - ../pm4-k8s-distribution/images/pm4-tools/pm wait-for-instance-ready - - name: Comment Instance - run: | - echo "Instance URL: '${INSTANCE_URL}'" - bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" - deleteEKS: - name: Delete Instance - if: github.event.action == 'closed' - runs-on: self-hosted - steps: - - name: Delete instance EKS - run: | - deploy=$(echo -n $IMAGE_TAG | md5sum | head -c 10) - echo "Deleting Instace :: ci-$deploy" - helm delete ci-$deploy - kubectl delete namespace ci-$deploy-ns-pm4 + - name: Checkout code + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID1 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY1 }} + aws-region: ${{ secrets.aws-region }} + + - name: Set up kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + - name: Authenticate with Amazon EKS + run: aws eks update-kubeconfig --region us-east-1 --name next-qa-cluster + - name: Deploy to Amazon EKS + run: kubectl get nodes + #imageEKS: + # name: build-docker-image-EKS + # if: github.event.action != 'closed' + # runs-on: ${{ vars.RUNNER }} + # steps: + # - name: Set image name + # run: | + # RESOLVED_IMAGE_TAG=${{ env.IMAGE_TAG }} + # echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:$RESOLVED_IMAGE_TAG" >> $GITHUB_ENV + # - name: Clone repo K8S + # run: | + # echo "IMAGE: ${{ env.IMAGE }}" + # git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + # - name: Generate image EKS + # run: | + # cd pm4-k8s-distribution/images + # branch=$CI_PACKAGE_BRANCH tag=${{env.IMAGE_TAG}} bash build.k8s-cicd.sh + # echo "VERSION=${{ env.IMAGE_TAG }}" >> $GITHUB_ENV + # - name: List Images + # run: | + # docker images + # - name: Run Trivy vulnerability scanner + # uses: aquasecurity/trivy-action@master + # with: + # image-ref: processmaker/enterprise:${{ env.VERSION }} + # format: 'table' + # exit-code: '0' + # ignore-unfixed: false + # vuln-type: 'os,library' + # scanners: 'vuln,secret' + # severity: 'MEDIUM,HIGH,CRITICAL' + # env: + # TRIVY_TIMEOUT: 30m + # - name: Login to Harbor + # uses: docker/login-action@v2 + # with: + # registry: ${{ secrets.REGISTRY_HOST }} + # username: ${{ secrets.REGISTRY_USERNAME }} + # password: ${{ secrets.REGISTRY_PASSWORD }} + # - name: Push Enterprise Image to Harbor + # run: | + # docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + # docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + #deployEKS: + # name: build-deploy-EKS + # #if: contains(github.event.pull_request.body, 'ci:deploy') + # #needs: imageEKS + # runs-on: self-hosted + # steps: + # - name: Clone private repository + # run: | + # git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd + # - name: CreateDB + # run: | + # cd argocd + # deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + # sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml + # kubectl get namespace ci-processmaker-ns-pm4 + # namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') + # kubectl apply -f template-db.yaml + # - name: Install pm4-tools + # run: | + # git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + # cd pm4-k8s-distribution/images/pm4-tools + # composer install --no-interaction + # cd .. + # - name: Deploy instance EKS + # run: | + # cd argocd + # deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + # current_datetime=$(echo -n ${{env.CURRENT_DATE}} | md5sum | head -c 10) + # echo "NAMESPACE : ci-$deploy-ns-pm4" + # helm repo add processmaker ${{ secrets.HELM_REPO }} --username ${{ secrets.HELM_USERNAME }} --password ${{ secrets.HELM_PASSWORD }} && helm repo update + # if ! kubectl get namespace/ci-$deploy-ns-pm4 ; then + # echo "Creating Deploy :: $deploy" + # sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml + # sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml + # cat template-instance.yaml + # helm install --timeout 0m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 + # sed -i "s/{{instance}}/ci-$deploy/" template-cert.yaml + # sed -i "s/{{tls-crt}}/${{ secrets.TLS_CRT }}/" template-cert.yaml + # sed -i "s/{{tls-key}}/${{ secrets.TLS_KEY }}/" template-cert.yaml + # kubectl apply -f template-cert.yaml + # else + # echo "Bouncing Instance "; + # sed -i "s/{{instance}}/ci-$deploy/g" template-bounce.yaml + # sed -i "s/{{current_datetime}}/$current_datetime/g" template-bounce.yaml + # helm upgrade --timeout 20m ci-$deploy processmaker/enterprise --version 2.1.0 + # kubectl apply -f template-bounce.yaml + # fi + # export INSTANCE_URL=https://ci-$deploy$DOM_EKS + # echo "INSTANCE_URL=${INSTANCE_URL}" >> "$GITHUB_ENV" + # ../pm4-k8s-distribution/images/pm4-tools/pm wait-for-instance-ready + # - name: Comment Instance + # run: | + # echo "Instance URL: '${INSTANCE_URL}'" + # bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" + #deleteEKS: + # name: Delete Instance + # if: github.event.action == 'closed' + # runs-on: self-hosted + # steps: + # - name: Delete instance EKS + # run: | + # deploy=$(echo -n $IMAGE_TAG | md5sum | head -c 10) + # echo "Deleting Instace :: ci-$deploy" + # helm delete ci-$deploy + # kubectl delete namespace ci-$deploy-ns-pm4 - runPhpUnit: - name: run-phpunit - if: github.event.action != 'closed' - needs: imageEKS - runs-on: ${{ vars.RUNNER }} - steps: - - name: Export Params - run: | - echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}}" >> $GITHUB_ENV - - name: Clone repo K8S - run: | - echo "IMAGE: ${{ env.IMAGE }}" - git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - - name: Login to Harbor - uses: docker/login-action@v2 - with: - registry: ${{ secrets.REGISTRY_HOST }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_PASSWORD }} - - name: PHPUnits - run: | - cd pm4-k8s-distribution/images/pm4-tools - docker pull $IMAGE - docker-compose down -v - docker-compose build phpunit - docker-compose run phpunit + #runPhpUnit: + # name: run-phpunit + # if: github.event.action != 'closed' + # needs: imageEKS + # runs-on: ${{ vars.RUNNER }} + # steps: + # - name: Export Params + # run: | + # echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}}" >> $GITHUB_ENV + # - name: Clone repo K8S + # run: | + # echo "IMAGE: ${{ env.IMAGE }}" + # git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + # - name: Login to Harbor + # uses: docker/login-action@v2 + # with: + # registry: ${{ secrets.REGISTRY_HOST }} + # username: ${{ secrets.REGISTRY_USERNAME }} + # password: ${{ secrets.REGISTRY_PASSWORD }} + # - name: PHPUnits + # run: | + # cd pm4-k8s-distribution/images/pm4-tools + # docker pull $IMAGE + # docker-compose down -v + # docker-compose build phpunit + # docker-compose run phpunit \ No newline at end of file From 051d78fea208bf871f5590db3d6394a744626121 Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 21:48:24 -0400 Subject: [PATCH 20/33] Add Second alternative --- .github/workflows/deploy-pm4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index c115d6383e..44a4cb78b6 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -47,7 +47,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID1 }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY1 }} - aws-region: ${{ secrets.aws-region }} + aws-region: ${{ secrets.AWS_REGION }} - name: Set up kubectl run: | From 34ca24a071784331d537ff4e9b3a6cf8b110a67e Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 22:14:30 -0400 Subject: [PATCH 21/33] Add Second alternative --- .github/workflows/deploy-pm4.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 44a4cb78b6..ce3a5bbe07 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -56,8 +56,10 @@ jobs: sudo mv kubectl /usr/local/bin/ - name: Authenticate with Amazon EKS run: aws eks update-kubeconfig --region us-east-1 --name next-qa-cluster - - name: Deploy to Amazon EKS - run: kubectl get nodes + #- name: Deploy to Amazon EKS + # run: kubectl get nodes + - name: Helm list + run: helm version && helm list #imageEKS: # name: build-docker-image-EKS # if: github.event.action != 'closed' From 3ed250a8a3a8ada02badc7907f6ef65b64941b67 Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 22:16:43 -0400 Subject: [PATCH 22/33] Add Second alternative --- .github/workflows/deploy-pm4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index ce3a5bbe07..ee2bb1c743 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -55,7 +55,7 @@ jobs: chmod +x kubectl sudo mv kubectl /usr/local/bin/ - name: Authenticate with Amazon EKS - run: aws eks update-kubeconfig --region us-east-1 --name next-qa-cluster + run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng #- name: Deploy to Amazon EKS # run: kubectl get nodes - name: Helm list From 494a7a0e32068d95880d60d79d9d51227c2b3d45 Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 23:00:54 -0400 Subject: [PATCH 23/33] Fix CICD --- .github/workflows/deploy-pm4.yml | 128 ++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index ee2bb1c743..269c957deb 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -3,11 +3,11 @@ run-name: ${{ github.actor }} send deploy EKS 🚀 on: pull_request: types: [opened, reopened, synchronize, edited, closed] - #schedule: - # - cron: '30 2 * * *' # every day at midnight push: branches: - FOUR-13153 + #schedule: + # - cron: '30 2 * * *' # every day at midnight workflow_dispatch: workflow_call: env: @@ -35,31 +35,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - deploy: - runs-on: ${{ vars.RUNNER }} - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID1 }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY1 }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Set up kubectl - run: | - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x kubectl - sudo mv kubectl /usr/local/bin/ - - name: Authenticate with Amazon EKS - run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng - #- name: Deploy to Amazon EKS - # run: kubectl get nodes - - name: Helm list - run: helm version && helm list #imageEKS: # name: build-docker-image-EKS # if: github.event.action != 'closed' @@ -103,10 +78,77 @@ jobs: # run: | # docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} # docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + deployEKS: + name: build-deploy-EKS + #if: contains(github.event.pull_request.body, 'ci:deploy') + #needs: imageEKS + runs-on: ${{ vars.RUNNER }} + steps: + - name: Clone private repository + run: | + git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd + - name: CreateDB + run: | + cd argocd + deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml + kubectl get namespace ci-processmaker-ns-pm4 + namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') + kubectl apply -f template-db.yaml + - name: Install pm4-tools + run: | + git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + cd pm4-k8s-distribution/images/pm4-tools + composer install --no-interaction + cd .. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID1 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY1 }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Set up kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + - name: Authenticate with Amazon EKS + run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng + - name: Deploy instance EKS + run: | + cd argocd + deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + current_datetime=$(echo -n ${{env.CURRENT_DATE}} | md5sum | head -c 10) + echo "NAMESPACE : ci-$deploy-ns-pm4" + helm repo add processmaker ${{ secrets.HELM_REPO }} --username ${{ secrets.HELM_USERNAME }} --password ${{ secrets.HELM_PASSWORD }} && helm repo update + if ! kubectl get namespace/ci-$deploy-ns-pm4 ; then + echo "Creating Deploy :: $deploy" + sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml + sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml + cat template-instance.yaml + helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 + sed -i "s/{{instance}}/ci-$deploy/" template-cert.yaml + sed -i "s/{{tls-crt}}/${{ secrets.TLS_CRT }}/" template-cert.yaml + sed -i "s/{{tls-key}}/${{ secrets.TLS_KEY }}/" template-cert.yaml + kubectl apply -f template-cert.yaml + else + echo "Bouncing Instance "; + sed -i "s/{{instance}}/ci-$deploy/g" template-bounce.yaml + sed -i "s/{{current_datetime}}/$current_datetime/g" template-bounce.yaml + helm upgrade --timeout 20m ci-$deploy processmaker/enterprise --version 2.1.0 + kubectl apply -f template-bounce.yaml + fi + export INSTANCE_URL=https://ci-$deploy$DOM_EKS + echo "INSTANCE_URL=${INSTANCE_URL}" >> "$GITHUB_ENV" + ../pm4-k8s-distribution/images/pm4-tools/pm wait-for-instance-ready + - name: Comment Instance + run: | + echo "Instance URL: '${INSTANCE_URL}'" + bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" #deployEKS: # name: build-deploy-EKS - # #if: contains(github.event.pull_request.body, 'ci:deploy') - # #needs: imageEKS + # if: contains(github.event.pull_request.body, 'ci:deploy') + # needs: imageEKS # runs-on: self-hosted # steps: # - name: Clone private repository @@ -138,11 +180,7 @@ jobs: # sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml # sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml # cat template-instance.yaml - # helm install --timeout 0m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 - # sed -i "s/{{instance}}/ci-$deploy/" template-cert.yaml - # sed -i "s/{{tls-crt}}/${{ secrets.TLS_CRT }}/" template-cert.yaml - # sed -i "s/{{tls-key}}/${{ secrets.TLS_KEY }}/" template-cert.yaml - # kubectl apply -f template-cert.yaml + # helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 # else # echo "Bouncing Instance "; # sed -i "s/{{instance}}/ci-$deploy/g" template-bounce.yaml @@ -157,17 +195,17 @@ jobs: # run: | # echo "Instance URL: '${INSTANCE_URL}'" # bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" - #deleteEKS: - # name: Delete Instance - # if: github.event.action == 'closed' - # runs-on: self-hosted - # steps: - # - name: Delete instance EKS - # run: | - # deploy=$(echo -n $IMAGE_TAG | md5sum | head -c 10) - # echo "Deleting Instace :: ci-$deploy" - # helm delete ci-$deploy - # kubectl delete namespace ci-$deploy-ns-pm4 + deleteEKS: + name: Delete Instance + if: github.event.action == 'closed' + runs-on: self-hosted + steps: + - name: Delete instance EKS + run: | + deploy=$(echo -n $IMAGE_TAG | md5sum | head -c 10) + echo "Deleting Instace :: ci-$deploy" + helm delete ci-$deploy + kubectl delete namespace ci-$deploy-ns-pm4 #runPhpUnit: # name: run-phpunit From 3970ae0c0e48724f46c632429e3d438c0440ab08 Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 23:04:33 -0400 Subject: [PATCH 24/33] Fix CICD --- .github/workflows/deploy-pm4.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 269c957deb..98080c5fd5 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -78,12 +78,9 @@ jobs: # run: | # docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} # docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} - deployEKS: - name: build-deploy-EKS - #if: contains(github.event.pull_request.body, 'ci:deploy') - #needs: imageEKS - runs-on: ${{ vars.RUNNER }} - steps: + prepareDB: + runs-on: self-hosted + steps: - name: Clone private repository run: | git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd @@ -94,7 +91,24 @@ jobs: sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml kubectl get namespace ci-processmaker-ns-pm4 namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') - kubectl apply -f template-db.yaml + kubectl apply -f template-db.yaml + deployEKS: + name: build-deploy-EKS + #if: contains(github.event.pull_request.body, 'ci:deploy') + #needs: imageEKS + runs-on: ${{ vars.RUNNER }} + steps: + - name: Clone private repository + run: | + git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd + #- name: CreateDB + # run: | + # cd argocd + # deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) + # sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml + # kubectl get namespace ci-processmaker-ns-pm4 + # namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') + # kubectl apply -f template-db.yaml - name: Install pm4-tools run: | git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution From 4917e8601ba7b50b6bd9c02e1360d0a538db994d Mon Sep 17 00:00:00 2001 From: mavalosn Date: Mon, 15 Jan 2024 23:30:08 -0400 Subject: [PATCH 25/33] Fix CICD --- .github/workflows/deploy-pm4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 98080c5fd5..efce6edd66 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -140,7 +140,7 @@ jobs: sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml cat template-instance.yaml - helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 + helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise sed -i "s/{{instance}}/ci-$deploy/" template-cert.yaml sed -i "s/{{tls-crt}}/${{ secrets.TLS_CRT }}/" template-cert.yaml sed -i "s/{{tls-key}}/${{ secrets.TLS_KEY }}/" template-cert.yaml From 0f476868289c11a2f4f893d1512bea02201da053 Mon Sep 17 00:00:00 2001 From: mavalosn Date: Tue, 16 Jan 2024 08:02:34 -0400 Subject: [PATCH 26/33] Test nes instance --- .github/workflows/deploy-pm4.yml | 90 ++++++++++++++++---------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index efce6edd66..218b4b70fa 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -18,7 +18,7 @@ env: pull_req_id: ${{github.event.pull_request.number}} DATE: $(date -d '-1 day' '+%Y-%m-%d'|sed 's/-//g') CURRENT_DATE: $(date '+%Y-%m-%d %H:%M:%S'|sed 's/-//g') - CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'FOUR-13151' }} + CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'FOUR-13153' }} CI_PROJECT: ${{github.event.pull_request.head.repo.name || 'processmaker' }} CI_PR_BODY: ${{ github.event_name == 'schedule' && 'No ci tags needed here' || github.event.pull_request.body }} IMAGE_TAG: $(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g") @@ -35,50 +35,51 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - #imageEKS: - # name: build-docker-image-EKS - # if: github.event.action != 'closed' - # runs-on: ${{ vars.RUNNER }} - # steps: - # - name: Set image name - # run: | - # RESOLVED_IMAGE_TAG=${{ env.IMAGE_TAG }} - # echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:$RESOLVED_IMAGE_TAG" >> $GITHUB_ENV - # - name: Clone repo K8S - # run: | - # echo "IMAGE: ${{ env.IMAGE }}" - # git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - # - name: Generate image EKS - # run: | - # cd pm4-k8s-distribution/images - # branch=$CI_PACKAGE_BRANCH tag=${{env.IMAGE_TAG}} bash build.k8s-cicd.sh - # echo "VERSION=${{ env.IMAGE_TAG }}" >> $GITHUB_ENV - # - name: List Images - # run: | - # docker images - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@master - # with: - # image-ref: processmaker/enterprise:${{ env.VERSION }} - # format: 'table' - # exit-code: '0' - # ignore-unfixed: false - # vuln-type: 'os,library' - # scanners: 'vuln,secret' - # severity: 'MEDIUM,HIGH,CRITICAL' - # env: - # TRIVY_TIMEOUT: 30m - # - name: Login to Harbor - # uses: docker/login-action@v2 - # with: - # registry: ${{ secrets.REGISTRY_HOST }} - # username: ${{ secrets.REGISTRY_USERNAME }} - # password: ${{ secrets.REGISTRY_PASSWORD }} - # - name: Push Enterprise Image to Harbor - # run: | - # docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} - # docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + imageEKS: + name: build-docker-image-EKS + if: github.event.action != 'closed' + runs-on: ${{ vars.RUNNER }} + steps: + - name: Set image name + run: | + RESOLVED_IMAGE_TAG=${{ env.IMAGE_TAG }} + echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:$RESOLVED_IMAGE_TAG" >> $GITHUB_ENV + - name: Clone repo K8S + run: | + echo "IMAGE: ${{ env.IMAGE }}" + git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + - name: Generate image EKS + run: | + cd pm4-k8s-distribution/images + branch=$CI_PACKAGE_BRANCH tag=${{env.IMAGE_TAG}} bash build.k8s-cicd.sh + echo "VERSION=${{ env.IMAGE_TAG }}" >> $GITHUB_ENV + - name: List Images + run: | + docker images + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: processmaker/enterprise:${{ env.VERSION }} + format: 'table' + exit-code: '0' + ignore-unfixed: false + vuln-type: 'os,library' + scanners: 'vuln,secret' + severity: 'MEDIUM,HIGH,CRITICAL' + env: + TRIVY_TIMEOUT: 30m + - name: Login to Harbor + uses: docker/login-action@v2 + with: + registry: ${{ secrets.REGISTRY_HOST }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Push Enterprise Image to Harbor + run: | + docker tag processmaker/enterprise:${{env.IMAGE_TAG}} ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} + docker push ${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}} prepareDB: + needs: imageEKS runs-on: self-hosted steps: - name: Clone private repository @@ -93,6 +94,7 @@ jobs: namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') kubectl apply -f template-db.yaml deployEKS: + needs: imageEKS name: build-deploy-EKS #if: contains(github.event.pull_request.body, 'ci:deploy') #needs: imageEKS From 8fdb8fb2f1d1242a645ade17b9f9f8a4e19c2a3f Mon Sep 17 00:00:00 2001 From: mavalosn Date: Tue, 16 Jan 2024 16:22:07 -0400 Subject: [PATCH 27/33] Fix CICD --- .github/workflows/deploy-pm4.yml | 125 ++++++++----------------------- 1 file changed, 31 insertions(+), 94 deletions(-) diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 218b4b70fa..9f2f048483 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -3,9 +3,6 @@ run-name: ${{ github.actor }} send deploy EKS 🚀 on: pull_request: types: [opened, reopened, synchronize, edited, closed] - push: - branches: - - FOUR-13153 #schedule: # - cron: '30 2 * * *' # every day at midnight workflow_dispatch: @@ -18,7 +15,7 @@ env: pull_req_id: ${{github.event.pull_request.number}} DATE: $(date -d '-1 day' '+%Y-%m-%d'|sed 's/-//g') CURRENT_DATE: $(date '+%Y-%m-%d %H:%M:%S'|sed 's/-//g') - CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'FOUR-13153' }} + CI_PACKAGE_BRANCH: ${{github.event.pull_request.head.ref || 'next' }} CI_PROJECT: ${{github.event.pull_request.head.repo.name || 'processmaker' }} CI_PR_BODY: ${{ github.event_name == 'schedule' && 'No ci tags needed here' || github.event.pull_request.body }} IMAGE_TAG: $(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g") @@ -94,23 +91,14 @@ jobs: namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') kubectl apply -f template-db.yaml deployEKS: - needs: imageEKS name: build-deploy-EKS - #if: contains(github.event.pull_request.body, 'ci:deploy') - #needs: imageEKS + if: contains(github.event.pull_request.body, 'ci:deploy') + needs: imageEKS runs-on: ${{ vars.RUNNER }} steps: - name: Clone private repository run: | - git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd - #- name: CreateDB - # run: | - # cd argocd - # deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) - # sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml - # kubectl get namespace ci-processmaker-ns-pm4 - # namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') - # kubectl apply -f template-db.yaml + git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd - name: Install pm4-tools run: | git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution @@ -161,56 +149,6 @@ jobs: run: | echo "Instance URL: '${INSTANCE_URL}'" bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" - #deployEKS: - # name: build-deploy-EKS - # if: contains(github.event.pull_request.body, 'ci:deploy') - # needs: imageEKS - # runs-on: self-hosted - # steps: - # - name: Clone private repository - # run: | - # git clone --depth 1 -b eng "https://$GITHUB_TOKEN@github.com/ProcessMaker/argocd.git" argocd - # - name: CreateDB - # run: | - # cd argocd - # deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) - # sed -i "s/{{instance}}/ci-$deploy/" template-db.yaml - # kubectl get namespace ci-processmaker-ns-pm4 - # namespace=$(kubectl get namespace $deploy-ns-pm4|grep $deploy|awk '{print $1}') - # kubectl apply -f template-db.yaml - # - name: Install pm4-tools - # run: | - # git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - # cd pm4-k8s-distribution/images/pm4-tools - # composer install --no-interaction - # cd .. - # - name: Deploy instance EKS - # run: | - # cd argocd - # deploy=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10) - # current_datetime=$(echo -n ${{env.CURRENT_DATE}} | md5sum | head -c 10) - # echo "NAMESPACE : ci-$deploy-ns-pm4" - # helm repo add processmaker ${{ secrets.HELM_REPO }} --username ${{ secrets.HELM_USERNAME }} --password ${{ secrets.HELM_PASSWORD }} && helm repo update - # if ! kubectl get namespace/ci-$deploy-ns-pm4 ; then - # echo "Creating Deploy :: $deploy" - # sed -i "s/{{instance}}/ci-$deploy/" template-instance.yaml - # sed -i "s/{{image}}/${{env.IMAGE_TAG}}/" template-instance.yaml - # cat template-instance.yaml - # helm install --timeout 40m -f template-instance.yaml ci-$deploy processmaker/enterprise --version 2.1.0 - # else - # echo "Bouncing Instance "; - # sed -i "s/{{instance}}/ci-$deploy/g" template-bounce.yaml - # sed -i "s/{{current_datetime}}/$current_datetime/g" template-bounce.yaml - # helm upgrade --timeout 20m ci-$deploy processmaker/enterprise --version 2.1.0 - # kubectl apply -f template-bounce.yaml - # fi - # export INSTANCE_URL=https://ci-$deploy$DOM_EKS - # echo "INSTANCE_URL=${INSTANCE_URL}" >> "$GITHUB_ENV" - # ../pm4-k8s-distribution/images/pm4-tools/pm wait-for-instance-ready - # - name: Comment Instance - # run: | - # echo "Instance URL: '${INSTANCE_URL}'" - # bash argocd/gh_comment.sh "$CI_PROJECT" "$pull_req_id" deleteEKS: name: Delete Instance if: github.event.action == 'closed' @@ -221,31 +159,30 @@ jobs: deploy=$(echo -n $IMAGE_TAG | md5sum | head -c 10) echo "Deleting Instace :: ci-$deploy" helm delete ci-$deploy - kubectl delete namespace ci-$deploy-ns-pm4 - - #runPhpUnit: - # name: run-phpunit - # if: github.event.action != 'closed' - # needs: imageEKS - # runs-on: ${{ vars.RUNNER }} - # steps: - # - name: Export Params - # run: | - # echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}}" >> $GITHUB_ENV - # - name: Clone repo K8S - # run: | - # echo "IMAGE: ${{ env.IMAGE }}" - # git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution - # - name: Login to Harbor - # uses: docker/login-action@v2 - # with: - # registry: ${{ secrets.REGISTRY_HOST }} - # username: ${{ secrets.REGISTRY_USERNAME }} - # password: ${{ secrets.REGISTRY_PASSWORD }} - # - name: PHPUnits - # run: | - # cd pm4-k8s-distribution/images/pm4-tools - # docker pull $IMAGE - # docker-compose down -v - # docker-compose build phpunit - # docker-compose run phpunit \ No newline at end of file + kubectl delete namespace ci-$deploy-ns-pm4 + runPhpUnit: + name: run-phpunit + if: github.event.action != 'closed' + needs: imageEKS + runs-on: ${{ vars.RUNNER }} + steps: + - name: Export Params + run: | + echo "IMAGE=${{ secrets.REGISTRY_HOST }}/processmaker/enterprise:${{env.IMAGE_TAG}}" >> $GITHUB_ENV + - name: Clone repo K8S + run: | + echo "IMAGE: ${{ env.IMAGE }}" + git clone --depth 1 -b "$K8S_BRANCH" "https://$GITHUB_TOKEN@github.com/ProcessMaker/pm4-k8s-distribution.git" pm4-k8s-distribution + - name: Login to Harbor + uses: docker/login-action@v2 + with: + registry: ${{ secrets.REGISTRY_HOST }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - name: PHPUnits + run: | + cd pm4-k8s-distribution/images/pm4-tools + docker pull $IMAGE + docker-compose down -v + docker-compose build phpunit + docker-compose run phpunit \ No newline at end of file From 5dfea307adfece2e1f4390cb1f014317e2ce3a22 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Wed, 17 Jan 2024 12:23:48 -0800 Subject: [PATCH 28/33] Update custom package --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 0d27b559e1..cf2cb79fef 100644 --- a/composer.json +++ b/composer.json @@ -98,7 +98,7 @@ "processmaker": { "build": "ad1d0941", "custom": { - "package-ellucian-ethos": "1.14.2", + "package-ellucian-ethos": "1.14.3", "package-plaid": "1.3.1", "paypal-package": "dev-fall", "pps-adsync": "dev-fall", From 06c65345e7cb82bdaf5cdc8705013bcbdb6ec46b Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Wed, 17 Jan 2024 12:24:42 -0800 Subject: [PATCH 29/33] Version 4.8.3 Build #a93a06a5 --- composer.json | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index cf2cb79fef..b34c6311c1 100644 --- a/composer.json +++ b/composer.json @@ -96,7 +96,7 @@ "Gmail" ], "processmaker": { - "build": "ad1d0941", + "build": "a93a06a5", "custom": { "package-ellucian-ethos": "1.14.3", "package-plaid": "1.3.1", diff --git a/composer.lock b/composer.lock index 2abfa40398..7fe1749e4a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5159980e43b36cd56a051795b1389dba", + "content-hash": "256b0e6faf785d88a800da3b927c6cba", "packages": [ { "name": "aws/aws-crt-php", From 1f80242d2b3e2e9a6c86be2774707baf38330f67 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Thu, 15 Feb 2024 09:23:04 -0800 Subject: [PATCH 30/33] Update custom package --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index b34c6311c1..14b798d99f 100644 --- a/composer.json +++ b/composer.json @@ -98,7 +98,7 @@ "processmaker": { "build": "a93a06a5", "custom": { - "package-ellucian-ethos": "1.14.3", + "package-ellucian-ethos": "1.14.4", "package-plaid": "1.3.1", "paypal-package": "dev-fall", "pps-adsync": "dev-fall", From 33b550af78d399614297e3f7dd7b67fa2b5ab2da Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Thu, 15 Feb 2024 09:23:56 -0800 Subject: [PATCH 31/33] Version 4.8.3 Build #afeeb7f4 --- composer.json | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 14b798d99f..bfd5183181 100644 --- a/composer.json +++ b/composer.json @@ -96,7 +96,7 @@ "Gmail" ], "processmaker": { - "build": "a93a06a5", + "build": "afeeb7f4", "custom": { "package-ellucian-ethos": "1.14.4", "package-plaid": "1.3.1", diff --git a/composer.lock b/composer.lock index 7fe1749e4a..a7554c8674 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "256b0e6faf785d88a800da3b927c6cba", + "content-hash": "ab2254443101bd1850fef0c1d82e3933", "packages": [ { "name": "aws/aws-crt-php", From 92e8f93742db0fef616e39b562d6ad35b4da8af2 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Fri, 16 Feb 2024 07:15:00 -0800 Subject: [PATCH 32/33] Update custom package --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index bfd5183181..0d6ea93156 100644 --- a/composer.json +++ b/composer.json @@ -98,7 +98,7 @@ "processmaker": { "build": "afeeb7f4", "custom": { - "package-ellucian-ethos": "1.14.4", + "package-ellucian-ethos": "1.14.5", "package-plaid": "1.3.1", "paypal-package": "dev-fall", "pps-adsync": "dev-fall", From eb63f2e411fa5100e202bc0a9a1c37cfc70d5863 Mon Sep 17 00:00:00 2001 From: Ryan Cooley Date: Fri, 16 Feb 2024 07:15:38 -0800 Subject: [PATCH 33/33] Version 4.8.3 Build #c8fb9ea3 --- composer.json | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 0d6ea93156..f3a24dbe44 100644 --- a/composer.json +++ b/composer.json @@ -96,7 +96,7 @@ "Gmail" ], "processmaker": { - "build": "afeeb7f4", + "build": "c8fb9ea3", "custom": { "package-ellucian-ethos": "1.14.5", "package-plaid": "1.3.1", diff --git a/composer.lock b/composer.lock index a7554c8674..498bf58d8f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ab2254443101bd1850fef0c1d82e3933", + "content-hash": "3e9b4ae75225ce29dfe125ce6a5b5e18", "packages": [ { "name": "aws/aws-crt-php",