Skip to content

Commit d8e0475

Browse files
fratzingerfratzinger
authored
feat: new util "chunkFind" (#23)
* feat: new util "chunkFind" * docs: add see refs to iterate/chunk
1 parent 9f75133 commit d8e0475

File tree

8 files changed

+245
-7
lines changed

8 files changed

+245
-7
lines changed

‎src/utils/chunk-find/chunk-find.util.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
title: chunkFind
3+
category: utils
4+
see: ["utils/iterateFind"]
5+
---

‎src/utils/chunk-find/chunk-find.util.test.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import type { Params } from '@feathersjs/feathers'
2+
import { feathers } from '@feathersjs/feathers'
3+
import { MemoryService } from '@feathersjs/memory'
4+
import { chunkFind } from './chunk-find.util.js'
5+
6+
type User = {
7+
id: number
8+
name: string
9+
}
10+
11+
const setup = async () => {
12+
const app = feathers<{
13+
users: MemoryService<
14+
User,
15+
Partial<User>,
16+
Params<{ id: number; name: string }>
17+
>
18+
}>()
19+
20+
app.use(
21+
'users',
22+
new MemoryService({
23+
id: 'id',
24+
startId: 1,
25+
multi: true,
26+
paginate: {
27+
default: 10,
28+
max: 100,
29+
},
30+
}),
31+
)
32+
33+
const usersService = app.service('users')
34+
35+
for (let i = 1; i <= 100; i++) {
36+
await usersService.create({ name: `test${i}` })
37+
}
38+
39+
return { app, usersService }
40+
}
41+
42+
describe('chunkFind', function () {
43+
it('basic usage', async function () {
44+
const { app } = await setup()
45+
46+
const chunks = []
47+
48+
for await (const chunk of chunkFind(app, 'users')) {
49+
chunks.push(chunk)
50+
}
51+
52+
expect(chunks).toHaveLength(10)
53+
expect(chunks[0]).toHaveLength(10)
54+
expect(chunks[0]![0]!.name).toBe('test1')
55+
expect(chunks[9]![9]!.name).toBe('test100')
56+
})
57+
58+
it('can skip items', async function () {
59+
const { app } = await setup()
60+
61+
const chunks = []
62+
63+
for await (const chunk of chunkFind(app, 'users', {
64+
params: { query: { $skip: 20 } },
65+
})) {
66+
chunks.push(chunk)
67+
}
68+
69+
expect(chunks).toHaveLength(8)
70+
expect(chunks[0]![0]!.name).toBe('test21')
71+
})
72+
73+
it('can set chunk size via $limit', async function () {
74+
const { app } = await setup()
75+
76+
const chunks = []
77+
78+
for await (const chunk of chunkFind(app, 'users', {
79+
params: { query: { $limit: 25 } },
80+
})) {
81+
chunks.push(chunk)
82+
}
83+
84+
expect(chunks).toHaveLength(4)
85+
expect(chunks[0]).toHaveLength(25)
86+
expect(chunks[3]).toHaveLength(25)
87+
})
88+
89+
it('can query for items', async function () {
90+
const { app } = await setup()
91+
92+
const chunks = []
93+
94+
for await (const chunk of chunkFind(app, 'users', {
95+
params: { query: { name: 'test1' } },
96+
})) {
97+
chunks.push(chunk)
98+
}
99+
100+
expect(chunks).toHaveLength(1)
101+
expect(chunks[0]).toEqual([expect.objectContaining({ name: 'test1' })])
102+
})
103+
104+
it("ignores paginate:false and always paginates", async function () {
105+
const { app } = await setup()
106+
107+
const chunks = []
108+
109+
for await (const chunk of chunkFind(app, 'users', {
110+
params: { query: { name: 'test1' }, paginate: false },
111+
})) {
112+
chunks.push(chunk)
113+
}
114+
115+
expect(chunks).toHaveLength(1)
116+
expect(chunks[0]).toEqual([expect.objectContaining({ name: 'test1' })])
117+
});
118+
})

‎src/utils/chunk-find/chunk-find.util.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import type { Application, Params } from '@feathersjs/feathers'
2+
import type { KeyOf } from '../../internal.utils.js'
3+
import type {
4+
InferFindParams,
5+
InferFindResultSingle,
6+
} from '../../utility-types/infer-service-methods.js'
7+
8+
type ChunkFindOptions<P extends Params = Params> = {
9+
params?: P
10+
}
11+
12+
/**
13+
* Use `for await` to iterate over chunks (pages) of results from a `find` method.
14+
*
15+
* This function is useful for processing large datasets in batches without loading everything into memory at once.
16+
* It uses pagination to fetch results in chunks, yielding each page's data array.
17+
*
18+
* @example
19+
* ```ts
20+
* import { chunkFind } from 'feathers-utils/utils'
21+
*
22+
* const app = feathers()
23+
*
24+
* // Assuming 'users' service has many records
25+
* for await (const users of chunkFind(app, 'users', {
26+
* params: { query: { active: true }, // Custom query parameters
27+
* } })) {
28+
* console.log(users) // Process each chunk of user records
29+
* }
30+
* ```
31+
*
32+
* @see https://utils.feathersjs.com/utils/chunk-find.html
33+
*/
34+
export async function* chunkFind<
35+
Services,
36+
Path extends KeyOf<Services>,
37+
Service extends Services[Path] = Services[Path],
38+
P extends Params = InferFindParams<Service>,
39+
Item = InferFindResultSingle<Service>,
40+
>(
41+
app: Application<Services>,
42+
servicePath: Path,
43+
options?: ChunkFindOptions<P>,
44+
): AsyncGenerator<Item[], void, unknown> {
45+
const service = app.service(servicePath)
46+
47+
if (!service || !('find' in service)) {
48+
throw new Error(`Service '${servicePath}' does not have a 'find' method.`)
49+
}
50+
51+
const params = {
52+
...options?.params,
53+
query: {
54+
...(options?.params?.query ?? {}),
55+
$limit: options?.params?.query?.$limit ?? 10,
56+
$skip: options?.params?.query?.$skip ?? 0,
57+
},
58+
paginate: {
59+
default: options?.params?.paginate?.default ?? 10,
60+
max: options?.params?.paginate?.max ?? 100,
61+
},
62+
}
63+
64+
let result
65+
66+
do {
67+
result = await (service as any).find(params)
68+
69+
yield result.data
70+
71+
params.query.$skip = (params.query.$skip ?? 0) + result.data.length
72+
} while (result.total > params.query.$skip)
73+
}

‎src/utils/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export * from './add-skip/add-skip.util.js'
2+
export * from './chunk-find/chunk-find.util.js'
23
export * from './add-to-query/add-to-query.util.js'
34
export * from './check-context/check-context.util.js'
45
export * from './context-to-json/context-to-json.util.js'

‎src/utils/iterate-find/iterate-find.util.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
---
22
title: iterateFind
33
category: utils
4+
see: ["utils/chunkFind"]
45
---

‎src/utils/iterate-find/iterate-find.util.test.ts‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ type User = {
88
name: string
99
}
1010

11+
const length = 1000;
12+
const max = 100;
13+
1114
const setup = async () => {
1215
const app = feathers<{
1316
users: MemoryService<
@@ -25,16 +28,16 @@ const setup = async () => {
2528
multi: true,
2629
paginate: {
2730
default: 10,
28-
max: 100,
31+
max,
2932
},
3033
}),
3134
)
3235

3336
const usersService = app.service('users')
3437

35-
for (let i = 1; i <= 100; i++) {
36-
await usersService.create({ name: `test${i}` })
37-
}
38+
const usersToCreate = Array.from({ length }).map((_, i) => ({ name: `test${i + 1}` }))
39+
40+
await usersService.create(usersToCreate)
3841

3942
return { app, usersService }
4043
}
@@ -50,7 +53,7 @@ describe('iterateFind', function () {
5053
}
5154

5255
expect(userNames).toEqual(
53-
Array.from({ length: 100 }).map((_, i) => `test${i + 1}`),
56+
Array.from({ length }).map((_, i) => `test${i + 1}`),
5457
)
5558
})
5659

@@ -66,7 +69,7 @@ describe('iterateFind', function () {
6669
}
6770

6871
expect(userNames).toEqual(
69-
Array.from({ length: 80 }).map((_, i) => `test${i + 21}`),
72+
Array.from({ length: length - 20 }).map((_, i) => `test${i + 21}`),
7073
)
7174
})
7275

@@ -83,4 +86,37 @@ describe('iterateFind', function () {
8386

8487
expect(userNames).toEqual(['test1'])
8588
})
89+
90+
it("ignores paginate:false and always paginates", async function () {
91+
const { app } = await setup()
92+
93+
const userNames = []
94+
95+
for await (const user of iterateFind(app, 'users', {
96+
params: { query: { name: 'test1' }, paginate: false },
97+
})) {
98+
userNames.push(user.name)
99+
}
100+
101+
expect(userNames).toEqual(['test1'])
102+
});
103+
104+
it("works with max", async function () {
105+
const { app } = await setup()
106+
107+
expect(max + 10).toBeLessThan(length)
108+
109+
const userNames = []
110+
111+
for await (const user of iterateFind(app, 'users', {
112+
params: { query: { $limit: max + 10 } },
113+
})) {
114+
userNames.push(user.name)
115+
}
116+
117+
expect(userNames).toEqual(
118+
Array.from({ length }).map((_, i) => `test${i + 1}`),
119+
)
120+
});
121+
86122
})

‎src/utils/iterate-find/iterate-find.util.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,12 @@ export async function* iterateFind<
5252
...options?.params,
5353
query: {
5454
...(options?.params?.query ?? {}),
55-
$limit: options?.params?.query?.$limit ?? 10,
55+
$limit: options?.params?.query?.$limit,
5656
$skip: options?.params?.query?.$skip ?? 0,
5757
},
58+
paginate: {
59+
default: options?.params?.paginate?.default ?? 10,
60+
},
5861
}
5962

6063
let result

‎test/index.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ const utils = [
4141
'addSkip',
4242
'addToQuery',
4343
'checkContext',
44+
'chunkFind',
4445
'contextToJson',
4546
'defineHooks',
4647
'getDataIsArray',

0 commit comments

Comments
 (0)