Skip to content

Commit 00978fd

Browse files
PratheepVPratheepV
authored
Algolio to Typesense (#51)
* Algolio to Typesense * Staging CD changes * indexr post script changes * build error * build error * build error
1 parent e5a071b commit 00978fd

10 files changed

Lines changed: 243 additions & 17783 deletions

File tree

‎.github/workflows/cd.yml‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,25 @@ jobs:
1616
node-version: '12'
1717
- name: Install dependencies
1818
run: |
19-
npm ci --ignore-scripts --unsafe-perm
19+
npm i --force --unsafe-perm
2020
- name: Force installing sharp
2121
run: npm install --ignore-scripts=false --verbose sharp@0.30.7
2222
- name: 'Create env file'
2323
run: |
2424
touch .env
25-
echo GATSBY_ALGOLIA_APP_ID=${{ secrets.GATSBY_ALGOLIA_APP_ID }} >> .env
26-
echo GATSBY_ALGOLIA_SEARCH_KEY=${{ secrets.GATSBY_ALGOLIA_SEARCH_KEY }} >> .env
27-
echo ALGOLIA_ADMIN_KEY=${{ secrets.ALGOLIA_ADMIN_KEY }} >> .env
2825
echo BUCKET_NAME=${{ secrets.BUCKET_NAME }} >> .env
2926
echo HOST_NAME=${{ secrets.HOST_NAME }} >> .env
3027
echo ASSET_HOST=${{ secrets.ASSET_HOST }} >> .env
3128
echo FRESHCHAT_TOKEN=${{ secrets.FRESHCHAT_TOKEN }} >> .env
3229
echo GA_TRACKING_ID=${{ secrets.GA_TRACKING_ID }} >> .env
3330
echo GTAG_TRACKING_ID_FOR_GA4=${{ secrets.GTAG_TRACKING_ID_FOR_GA4 }} >> .env
3431
echo GTAG_TRACKING_ID_FOR_GTAG=${{ secrets.GTAG_TRACKING_ID_FOR_GTAG }} >> .env
32+
echo TYPESENSE_HOST=${{ secrets.TYPESENSE_HOST }} >> .env
33+
echo TYPESENSE_PORT=${{ secrets.TYPESENSE_PORT }} >> .env
34+
echo TYPESENSE_PROTOCOL=${{ secrets.TYPESENSE_PROTOCOL }} >> .env
35+
echo TYPESENSE_COLLECTION=${{ secrets.TYPESENSE_COLLECTION }} >> .env
36+
echo TYPESENSE_API_KEY=${{ secrets.TYPESENSE_API_KEY }} >> .env
37+
echo TYPESENSE_SEARCH_API_KEY=${{ secrets.TYPESENSE_SEARCH_API_KEY }} >> .env
3538
cat .env
3639
- name: Build
3740
run: |
@@ -41,4 +44,35 @@ jobs:
4144
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
4245
AWS_DEFAULT_REGION: 'us-east-1'
4346
AWS_DEFAULT_OUTPUT: json
47+
TYPESENSE_HOST: ${{ secrets.TYPESENSE_HOST }}
48+
TYPESENSE_PORT: ${{ secrets.TYPESENSE_PORT }}
49+
TYPESENSE_PROTOCOL: ${{ secrets.TYPESENSE_PROTOCOL }}
50+
TYPESENSE_COLLECTION: ${{ secrets.TYPESENSE_COLLECTION }}
51+
TYPESENSE_API_KEY: ${{ secrets.TYPESENSE_API_KEY }}
52+
TYPESENSE_SEARCH_API_KEY: ${{ secrets.TYPESENSE_SEARCH_API_KEY }}
53+
54+
- name: Start Gatsby server
55+
run: npm run develop &
56+
env:
57+
GATSBY_TELEMETRY_DISABLED: 1
58+
59+
- name: Wait for Gatsby server to start
60+
run: |
61+
sleep 10
62+
until $(curl --output /dev/null --silent --head --fail http://localhost:8001); do
63+
sleep 2
64+
done
65+
66+
- name: Run indexing script
67+
run: npm run postbuild
68+
env:
69+
TYPESENSE_HOST: ${{ secrets.TYPESENSE_HOST }}
70+
TYPESENSE_PORT: ${{ secrets.TYPESENSE_PORT }}
71+
TYPESENSE_PROTOCOL: ${{ secrets.TYPESENSE_PROTOCOL }}
72+
TYPESENSE_COLLECTION: ${{ secrets.TYPESENSE_COLLECTION }}
73+
TYPESENSE_API_KEY: ${{ secrets.TYPESENSE_API_KEY }}
74+
TYPESENSE_SEARCH_API_KEY: ${{ secrets.TYPESENSE_SEARCH_API_KEY }}
75+
76+
- name: Stop Gatsby server
77+
run: lsof -i :8001 -t | xargs kill
4478

‎.github/workflows/ci.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
node-version: '12'
1616
- name: Install dependencies
1717
run: |
18-
npm ci --ignore-scripts --unsafe-perm
18+
npm i --force --unsafe-perm
1919
- name: Force installing sharp
2020
run: npm install --ignore-scripts=false --verbose sharp@0.30.7
2121
- name: 'Create env file'
@@ -31,6 +31,12 @@ jobs:
3131
echo GA_TRACKING_ID=${{ secrets.GA_TRACKING_ID }} >> .env
3232
echo GTAG_TRACKING_ID_FOR_GA4=${{ secrets.GTAG_TRACKING_ID_FOR_GA4 }} >> .env
3333
echo GTAG_TRACKING_ID_FOR_GTAG=${{ secrets.GTAG_TRACKING_ID_FOR_GTAG }} >> .env
34+
echo TYPESENSE_HOST=${{ secrets.WEBSITE_STAGING_TYPESENSE_HOST }} >> .env
35+
echo TYPESENSE_PORT=${{ secrets.WEBSITE_STAGING_TYPESENSE_PORT }} >> .env
36+
echo TYPESENSE_PROTOCOL=${{ secrets.WEBSITE_STAGING_TYPESENSE_PROTOCOL }} >> .env
37+
echo TYPESENSE_COLLECTION=${{ secrets.WEBSITE_STAGING_TYPESENSE_COLLECTION }} >> .env
38+
echo TYPESENSE_API_KEY=${{ secrets.WEBSITE_STAGING_TYPESENSE_API_KEY }} >> .env
39+
echo TYPESENSE_SEARCH_API_KEY=${{ secrets.WEBSITE_STAGING_TYPESENSE_SEARCH_API_KEY }} >> .env
3440
cat .env
3541
- name: Build
3642
run: |

‎.github/workflows/staging-cd.yml‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,19 @@ jobs:
2222
- name: 'Create env file'
2323
run: |
2424
touch .env
25-
echo GATSBY_ALGOLIA_APP_ID=${{ secrets.WEBSITE_STAGING_GATSBY_ALGOLIA_APP_ID }} >> .env
26-
echo GATSBY_ALGOLIA_SEARCH_KEY=${{ secrets.WEBSITE_STAGING_GATSBY_ALGOLIA_SEARCH_KEY }} >> .env
27-
echo ALGOLIA_ADMIN_KEY=${{ secrets.WEBSITE_STAGING_ALGOLIA_ADMIN_KEY }} >> .env
2825
echo BUCKET_NAME=${{ secrets.WEBSITE_STAGING_BUCKET_NAME }} >> .env
2926
echo HOST_NAME=${{ secrets.WEBSITE_STAGING_HOST_NAME }} >> .env
3027
echo ASSET_HOST=${{ secrets.WEBSITE_STAGING_ASSET_HOST }} >> .env
3128
echo FRESHCHAT_TOKEN=${{ secrets.WEBSITE_STAGING_FRESHCHAT_TOKEN }} >> .env
3229
echo GA_TRACKING_ID=${{ secrets.WEBSITE_STAGING_GA_TRACKING_ID }} >> .env
3330
echo GTAG_TRACKING_ID_FOR_GA4=${{ secrets.WEBSITE_STAGING_GTAG_TRACKING_ID_FOR_GA4 }} >> .env
3431
echo GTAG_TRACKING_ID_FOR_GTAG=${{ secrets.WEBSITE_STAGING_GTAG_TRACKING_ID_FOR_GTAG }} >> .env
32+
echo TYPESENSE_HOST=${{ secrets.WEBSITE_STAGING_TYPESENSE_HOST }} >> .env
33+
echo TYPESENSE_PORT=${{ secrets.WEBSITE_STAGING_TYPESENSE_PORT }} >> .env
34+
echo TYPESENSE_PROTOCOL=${{ secrets.WEBSITE_STAGING_TYPESENSE_PROTOCOL }} >> .env
35+
echo TYPESENSE_COLLECTION=${{ secrets.WEBSITE_STAGING_TYPESENSE_COLLECTION }} >> .env
36+
echo TYPESENSE_API_KEY=${{ secrets.WEBSITE_STAGING_TYPESENSE_API_KEY }} >> .env
37+
echo TYPESENSE_SEARCH_API_KEY=${{ secrets.WEBSITE_STAGING_TYPESENSE_SEARCH_API_KEY }} >> .env
3538
cat .env
3639
- name: Set environment
3740
run : sed -i 's/localhost/website.testsigma.com/g' src/environment.ts
@@ -43,3 +46,38 @@ jobs:
4346
AWS_SECRET_ACCESS_KEY: ${{ secrets.WEBSITE_STAGING_AWS_SECRET_ACCESS_KEY }}
4447
AWS_DEFAULT_REGION: 'us-east-1'
4548
AWS_DEFAULT_OUTPUT: json
49+
TYPESENSE_HOST: ${{ secrets.WEBSITE_STAGING_TYPESENSE_HOST }}
50+
TYPESENSE_PORT: ${{ secrets.WEBSITE_STAGING_TYPESENSE_PORT }}
51+
TYPESENSE_PROTOCOL: ${{ secrets.WEBSITE_STAGING_TYPESENSE_PROTOCOL }}
52+
TYPESENSE_COLLECTION: ${{ secrets.WEBSITE_STAGING_TYPESENSE_COLLECTION }}
53+
TYPESENSE_API_KEY: ${{ secrets.WEBSITE_STAGING_TYPESENSE_API_KEY }}
54+
TYPESENSE_SEARCH_API_KEY: ${{ secrets.WEBSITE_STAGING_TYPESENSE_SEARCH_API_KEY }}
55+
56+
- name: Start Gatsby server
57+
run: npm run develop &
58+
env:
59+
GATSBY_TELEMETRY_DISABLED: 1
60+
61+
- name: Wait for Gatsby server to start
62+
run: |
63+
sleep 10
64+
until $(curl --output /dev/null --silent --head --fail http://localhost:8001); do
65+
sleep 2
66+
done
67+
68+
- name: Run indexing script
69+
run: npm run postbuild
70+
env:
71+
AWS_ACCESS_KEY_ID: ${{ secrets.WEBSITE_STAGING_AWS_ACCESS_KEY_ID }}
72+
AWS_SECRET_ACCESS_KEY: ${{ secrets.WEBSITE_STAGING_AWS_SECRET_ACCESS_KEY }}
73+
AWS_DEFAULT_REGION: 'us-east-1'
74+
AWS_DEFAULT_OUTPUT: json
75+
TYPESENSE_HOST: ${{ secrets.WEBSITE_STAGING_TYPESENSE_HOST }}
76+
TYPESENSE_PORT: ${{ secrets.WEBSITE_STAGING_TYPESENSE_PORT }}
77+
TYPESENSE_PROTOCOL: ${{ secrets.WEBSITE_STAGING_TYPESENSE_PROTOCOL }}
78+
TYPESENSE_COLLECTION: ${{ secrets.WEBSITE_STAGING_TYPESENSE_COLLECTION }}
79+
TYPESENSE_API_KEY: ${{ secrets.WEBSITE_STAGING_TYPESENSE_API_KEY }}
80+
TYPESENSE_SEARCH_API_KEY: ${{ secrets.WEBSITE_STAGING_TYPESENSE_SEARCH_API_KEY }}
81+
82+
- name: Stop Gatsby server
83+
run: lsof -i :8001 -t | xargs kill

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ public
44
.idea/*
55
*.iml
66
.env
7-
.DS_Store
7+
.DS_Store
8+
package-lock.json

‎.vscode/launch.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
// Use IntelliSense to learn about possible attributes.
3+
// Hover to view descriptions of existing attributes.
4+
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
5+
"version": "0.2.0",
6+
"configurations": [
7+
{
8+
"type": "node",
9+
"request": "launch",
10+
"name": "Tutorials",
11+
"skipFiles": [
12+
"<node_internals>/**"
13+
],
14+
"cwd": "${workspaceRoot}",
15+
"runtimeExecutable": "/usr/local/bin/npm",
16+
"runtimeArgs": [
17+
"run-script", "develop"
18+
],
19+
"outputCapture": "std",
20+
}
21+
]
22+
}

‎gatsby-config.js‎

Lines changed: 6 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -3,58 +3,6 @@ require('dotenv').config({
33
path: `.env`,
44
});
55

6-
const pageQuery = `{
7-
docs: allMarkdownRemark(
8-
filter: {
9-
fileAbsolutePath: { regex: "/tutorials/" },
10-
}
11-
) {
12-
edges {
13-
node {
14-
headings(depth: h3) {
15-
value
16-
}
17-
objectID: id
18-
frontmatter {
19-
title
20-
search_keyword
21-
contextual_links {
22-
type
23-
name
24-
url
25-
}
26-
}
27-
fields {
28-
slug
29-
}
30-
excerpt(
31-
pruneLength: 6700
32-
)
33-
}
34-
}
35-
}
36-
}`;
37-
38-
39-
function pageToAlgoliaRecord({ node: { id, frontmatter, ...rest } }) {
40-
return {
41-
objectID: id,
42-
...frontmatter,
43-
...rest,
44-
};
45-
}
46-
47-
const settings = { attributesToSnippet: ['excerpt:20'] };
48-
49-
const queries = [
50-
{
51-
query: pageQuery,
52-
transformer: ({ data }) => data.docs.edges.map(pageToAlgoliaRecord),
53-
indexName: 'OS Docs',
54-
settings,
55-
},
56-
];
57-
586

597
module.exports = {
608
assetPrefix: process.env.ASSET_HOST,
@@ -88,18 +36,6 @@ module.exports = {
8836
path: `${__dirname}/src/`,
8937
},
9038
},
91-
{
92-
resolve: 'gatsby-plugin-algolia',
93-
options: {
94-
appId: process.env.GATSBY_ALGOLIA_APP_ID,
95-
apiKey: process.env.ALGOLIA_ADMIN_KEY,
96-
queries,
97-
chunkSize: 10000, // default: 1000
98-
enablePartialUpdates: true, // only index new, changed, deleted records
99-
matchFields: ['excerpt', 'contextual_links', 'search_keyword', 'headings', 'fields', 'modified'],
100-
concurrentQueries: false,
101-
},
102-
},
10339
{
10440
resolve: `gatsby-plugin-ts-freshchat`,
10541
options: {
@@ -167,6 +103,12 @@ module.exports = {
167103
},
168104
},
169105
},
106+
},
107+
{
108+
resolve: 'gatsby-plugin-env-variables',
109+
options: {
110+
allowList: ['TYPESENSE_HOST', 'TYPESENSE_PORT', 'TYPESENSE_PROTOCOL', 'TYPESENSE_API_KEY', "TYPESENSE_SEARCH_API_KEY", "TYPESENSE_COLLECTION"],
111+
},
170112
}
171113
],
172114
},

0 commit comments

Comments
 (0)