From e567d0ac7a34fc238cd2eb993de53d0bbd406d77 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Wed, 13 Nov 2024 20:34:13 -0700 Subject: [PATCH 1/5] Exclude repos/projects by glob pattern --- .gitignore | 8 +++++++- packages/backend/src/utils.test.ts | 32 +++++++++++++++++++++++++++++- packages/backend/src/utils.ts | 7 ++++--- schemas/v2/index.json | 8 ++++---- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index d62fb7927..6edaf9094 100644 --- a/.gitignore +++ b/.gitignore @@ -160,7 +160,13 @@ dist # End of https://www.toptal.com/developers/gitignore/api/yarn,node +# IDE (intellij) +.idea + .sourcebot /bin /config.json -.DS_Store \ No newline at end of file +.DS_Store +*.iml +package-lock.json + diff --git a/packages/backend/src/utils.test.ts b/packages/backend/src/utils.test.ts index 84d77f21f..db7f5023a 100644 --- a/packages/backend/src/utils.test.ts +++ b/packages/backend/src/utils.test.ts @@ -1,5 +1,35 @@ import { expect, test } from 'vitest'; -import { arraysEqualShallow, isRemotePath } from './utils'; +import { arraysEqualShallow, isRemotePath, excludeReposByName } from './utils'; +import { GitRepository, Repository } from './types'; + +const testNames: string[] = [ + "abcdefg/zfmno/ioiwerj/fawdf", + "abcdefg/zfmno/ioiwerj/werw", + "abcdefg/zfmno/ioiwerj/terne", + "abcdefg/zfmno/ioiwerj/asdf45e4r", + "abcdefg/zfmno/ioiwerj/ddee", + "abcdefg/zfmno/ioiwerj/ccdfeee", + "abcdefg/zfmno/sadfaw", + "abcdefg/zfmno/ioiwerj/wwe", + "abcdefg/ieieiowowieu8383/ieckup-e", + "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3", + "abcdefg/zfmno/sreerew"]; + +const createRepository = (name: string) => ({ + vcs: 'git', + id: name, + name: name, + path: name, + isStale: false, + cloneUrl: name, + branches: [name], + tags: [name] +}); + +test('should filter repos by micromatch pattern', () => { + const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/zfmno/**']); + expect(filteredRepos.length).toBe(2); +}); test('should return true for identical arrays', () => { expect(arraysEqualShallow([1, 2, 3], [1, 2, 3])).toBe(true); diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 8ac58d788..8996dd1e3 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,6 +1,7 @@ import { Logger } from "winston"; import { AppContext, Repository } from "./types.js"; import path from 'path'; +import micromatch from "micromatch"; export const measure = async (cb : () => Promise) => { const start = Date.now(); @@ -36,10 +37,10 @@ export const excludeArchivedRepos = (repos: T[], logger?: }); } + export const excludeReposByName = (repos: T[], excludedRepoNames: string[], logger?: Logger) => { - const excludedRepos = new Set(excludedRepoNames); return repos.filter((repo) => { - if (excludedRepos.has(repo.name)) { + if (micromatch.isMatch(repo.name, excludedRepoNames)) { logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.repos contains ${repo.name}`); return false; } @@ -90,4 +91,4 @@ export const arraysEqualShallow = (a?: readonly T[], b?: readonly T[]) => { } return true; -} \ No newline at end of file +} diff --git a/schemas/v2/index.json b/schemas/v2/index.json index 256fd3cdc..009495ec7 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -147,7 +147,7 @@ "type": "array", "items": { "type": "string", - "pattern": "^[\\w.-]+\\/[\\w.-]+$" + "pattern": "^[\\w.-]+(\\/[\\w.-]+)*$" }, "default": [], "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." @@ -239,7 +239,7 @@ "type": "array", "items": { "type": "string", - "pattern": "^[\\w.-]+\\/[\\w.-]+$" + "pattern": "^[\\w.-]+(\\/[\\w.-]+)*$" }, "default": [], "examples": [ @@ -337,7 +337,7 @@ "type": "array", "items": { "type": "string", - "pattern": "^[\\w.-]+\\/[\\w.-]+$" + "pattern": "^[\\w.-]+(\\/[\\w.-]+)*$" }, "default": [], "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." @@ -432,4 +432,4 @@ } }, "additionalProperties": false -} \ No newline at end of file +} From 168af2e806ac4968d17f5adbca68adc8354902f6 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Wed, 13 Nov 2024 20:35:47 -0700 Subject: [PATCH 2/5] Changed .gitignore back --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index 6edaf9094..66fa4405a 100644 --- a/.gitignore +++ b/.gitignore @@ -160,13 +160,8 @@ dist # End of https://www.toptal.com/developers/gitignore/api/yarn,node -# IDE (intellij) -.idea - .sourcebot /bin /config.json .DS_Store -*.iml -package-lock.json From bb09d64ec41acb6248f1865e0a8e2fc773ea1b84 Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Wed, 13 Nov 2024 20:38:44 -0700 Subject: [PATCH 3/5] removed .gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 66fa4405a..d62fb7927 100644 --- a/.gitignore +++ b/.gitignore @@ -163,5 +163,4 @@ dist .sourcebot /bin /config.json -.DS_Store - +.DS_Store \ No newline at end of file From 2f09a1d6303798c870e5f098a63b26f71adf477b Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Wed, 13 Nov 2024 20:39:52 -0700 Subject: [PATCH 4/5] Optimized imports --- packages/backend/src/utils.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/utils.test.ts b/packages/backend/src/utils.test.ts index db7f5023a..be6c432c0 100644 --- a/packages/backend/src/utils.test.ts +++ b/packages/backend/src/utils.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import { arraysEqualShallow, isRemotePath, excludeReposByName } from './utils'; -import { GitRepository, Repository } from './types'; +import { Repository } from './types'; const testNames: string[] = [ "abcdefg/zfmno/ioiwerj/fawdf", From fa1f8af48fce8ef229a3f8401c0806aef1cd4aca Mon Sep 17 00:00:00 2001 From: Steven Nagie Date: Thu, 14 Nov 2024 13:43:46 -0700 Subject: [PATCH 5/5] Added glob pattern examples to filter.json, removed pattern requirement and updated description for exclude repos/projects in schema json, improved existing micromatch test and added test for name exact match --- configs/filter.json | 12 +++++++++--- packages/backend/src/utils.test.ts | 31 ++++++++++++++++++++++++++++-- schemas/v2/index.json | 15 ++++++--------- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/configs/filter.json b/configs/filter.json index 1e31d5243..ca6bd2831 100644 --- a/configs/filter.json +++ b/configs/filter.json @@ -15,7 +15,9 @@ "forks": true, "repos": [ "my-org/repo1", - "my-org/repo2" + "my-org/repo2", + "my-org/sub-org-1/**", + "my-org/sub-org-*/**" ] } }, @@ -34,7 +36,9 @@ "forks": true, "projects": [ "my-group/project1", - "my-group/project2" + "my-group/project2", + "my-org/sub-org-1/**", + "my-org/sub-org-*/**" ] } }, @@ -53,7 +57,9 @@ "forks": true, "repos": [ "my-org/repo1", - "my-org/repo2" + "my-org/repo2", + "my-org/sub-org-1/**", + "my-org/sub-org-*/**" ] } }, diff --git a/packages/backend/src/utils.test.ts b/packages/backend/src/utils.test.ts index be6c432c0..76c140678 100644 --- a/packages/backend/src/utils.test.ts +++ b/packages/backend/src/utils.test.ts @@ -12,8 +12,8 @@ const testNames: string[] = [ "abcdefg/zfmno/sadfaw", "abcdefg/zfmno/ioiwerj/wwe", "abcdefg/ieieiowowieu8383/ieckup-e", - "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3", - "abcdefg/zfmno/sreerew"]; + "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3" +]; const createRepository = (name: string) => ({ vcs: 'git', @@ -27,8 +27,35 @@ const createRepository = (name: string) => ({ }); test('should filter repos by micromatch pattern', () => { + // bad glob patterns + const unfilteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['/zfmno/']); + expect(unfilteredRepos.length).toBe(10); + expect(unfilteredRepos.map(r => r.name)).toEqual(testNames); + const unfilteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), ['**zfmno**']); + expect(unfilteredRepos1.length).toBe(10); + expect(unfilteredRepos1.map(r => r.name)).toEqual(testNames); + + // good glob patterns const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/zfmno/**']); expect(filteredRepos.length).toBe(2); + expect(filteredRepos.map(r => r.name)).toEqual(["abcdefg/ieieiowowieu8383/ieckup-e", "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"]); + const filteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/*fmn*/**']); + expect(filteredRepos1.length).toBe(2); + expect(filteredRepos1.map(r => r.name)).toEqual(["abcdefg/ieieiowowieu8383/ieckup-e", "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"]); +}); + +test('should filter repos by name exact match', () => { + const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), testNames.slice(1, 9)); + expect(filteredRepos.length).toBe(2); + expect(filteredRepos.map(r => r.name)).toEqual([testNames[0], testNames[9]]); + + const filteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), testNames.slice(3, 5)); + expect(filteredRepos1.length).toBe(8); + expect(filteredRepos1.map(r => r.name)).toEqual([testNames[0], testNames[1], testNames[2], testNames[5], testNames[6], testNames[7], testNames[8], testNames[9]]); + + const filteredRepos2 = excludeReposByName(testNames.map(n => (createRepository(n))), [testNames[0], testNames[7], testNames[9]]); + expect(filteredRepos2.length).toBe(7); + expect(filteredRepos2.map(r => r.name)).toEqual([...testNames.slice(1, 7), testNames[8]]); }); test('should return true for identical arrays', () => { diff --git a/schemas/v2/index.json b/schemas/v2/index.json index 009495ec7..e71058d2d 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -146,11 +146,10 @@ "repos": { "type": "array", "items": { - "type": "string", - "pattern": "^[\\w.-]+(\\/[\\w.-]+)*$" + "type": "string" }, "default": [], - "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." } }, "additionalProperties": false @@ -238,8 +237,7 @@ "projects": { "type": "array", "items": { - "type": "string", - "pattern": "^[\\w.-]+(\\/[\\w.-]+)*$" + "type": "string" }, "default": [], "examples": [ @@ -247,7 +245,7 @@ "my-group/my-project" ] ], - "description": "List of individual projects to exclude from syncing. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/" + "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/" } }, "additionalProperties": false @@ -336,11 +334,10 @@ "repos": { "type": "array", "items": { - "type": "string", - "pattern": "^[\\w.-]+(\\/[\\w.-]+)*$" + "type": "string" }, "default": [], - "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." } }, "additionalProperties": false