Skip to content
This repository was archived by the owner on Mar 22, 2020. It is now read-only.

Commit e5df66b

Browse files
iliakaniliakan
committed
add contribs stats
1 parent 93f5cef commit e5df66b

17 files changed

Lines changed: 547 additions & 27 deletions

‎app.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const router = new Router();
99

1010
router.post('/hook', require('./handlers/hook').post);
1111
router.get('/stats', require('./handlers/stats').get);
12+
router.get('/contributors', require('./handlers/contributors').get);
1213
router.get('/stats/:lang.svg', require('./handlers/statsInSvg').get);
1314
router.get('/cache-test', require('./handlers/cacheTest').get);
1415

‎bin/refresh.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@ const updateRepo = require('../lib/updateRepo');
1717
for (let repoName in config.secret.repos) {
1818
let repoConfig = config.secret.repos[repoName];
1919

20-
await Stats.instance().count(repoName);
20+
// fixme
21+
// if (repoConfig.lang !== 'zh') continue;
22+
23+
await Stats.instance().gather(repoName);
2124
}
2225

2326
})().catch(console.error);

‎githubEmailMap.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"////": "Map of email to github user",
3+
"1342878298@qq.com" : "Starriers",
4+
"sunhaokk@qq.com": "sunhaokk",
5+
"sqrtthree@foxmail.com": "sqrtthree",
6+
"espoir.ka@gmail.com": "kenjii",
7+
"nakaken88888888@gmail.com": "nakaken88",
8+
"usernamehw@gmail.com": "usernamehw",
9+
"imidom@gmail.com": "imidom",
10+
"26034704+a-ogilvie@users.noreply.github.com": "a-ogilvie",
11+
"jeffrymbothe@gmail.com": "jmbothe",
12+
"simmayor@gmail.com": "simmayor",
13+
"davegregg@gmail.com": "davegregg",
14+
"iketari@gmail.com": "iketari",
15+
"mail@igoradamenko.com": "igoradamenko",
16+
"18657532086@163.com": "steinliber"
17+
}

‎handlers/contributors.js‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
const config = require('../config');
2+
const debug = require('debug')('handlers:stats');
3+
const Stats = require('../lib/stats');
4+
const { repos } = config.secret;
5+
let {mapToObj} = require('../lib/util');
6+
7+
exports.get = async function (ctx) {
8+
9+
const { lang } = ctx.request.query;
10+
11+
debug('LANG', lang);
12+
13+
let repoName, repo;
14+
15+
let found = false;
16+
for ([repoName, repo] of Object.entries(repos)) {
17+
if (repo.lang === lang) {
18+
found = true;
19+
break;
20+
}
21+
}
22+
23+
if (!found) {
24+
return; // 404
25+
}
26+
27+
debug('repo', repoName, repo);
28+
29+
const {files, emailToGithubUser} = Stats.instance().get(repoName).contributors;
30+
31+
let statsByAuthor = Object.create(null);
32+
33+
// console.log(files);
34+
35+
for(let file in files) {
36+
let fileStats = files[file];
37+
for (let author in fileStats) {
38+
statsByAuthor[author] = (statsByAuthor[author] || 0) + fileStats[author];
39+
}
40+
}
41+
42+
// remote iliakan@gmail.com, don't count original content author
43+
// all source code has him as the author, that's a lot already
44+
if (lang !== 'en' && lang !== 'ru') {
45+
delete statsByAuthor['iliakan@gmail.com']; // (actually email below is used)
46+
delete statsByAuthor['iliakan@users.noreply.github.com'];
47+
}
48+
49+
let linesTotal = 0;
50+
for (let linesCount of Object.values(statsByAuthor)) {
51+
linesTotal += linesCount;
52+
}
53+
54+
let result = new Map();
55+
56+
// console.log("stats by author", statsByAuthor);
57+
58+
for (let githubEmail in statsByAuthor) {
59+
let linesCount = statsByAuthor[githubEmail];
60+
61+
if (linesCount < 10) {
62+
// skip contribs who have less than 10 lines
63+
continue;
64+
}
65+
66+
let githubUser = emailToGithubUser[githubEmail];
67+
68+
if (githubUser) {
69+
// we know github user with that email
70+
// let's group all results from other his emails under one entry
71+
if (result.has(githubUser.login)) {
72+
result.get(githubUser.login).linesCount += linesCount;
73+
} else {
74+
result.set(githubUser.login, {
75+
githubEmail,
76+
linesCount,
77+
githubUser
78+
});
79+
}
80+
} else {
81+
// email is the key if we don't know such user
82+
result.set(githubEmail, {
83+
githubEmail,
84+
linesCount,
85+
githubUser
86+
});
87+
}
88+
}
89+
90+
91+
for (let value of result.values()) {
92+
value.percent = (value.linesCount / linesTotal * 100).toFixed(2);
93+
}
94+
95+
// console.log(result.entries());
96+
result = new Map(Array.from(result).sort((a, b) => b[1].linesCount - a[1].linesCount));
97+
98+
// console.log(result);
99+
100+
ctx.body = mapToObj(result);
101+
102+
};

‎handlers/hook.js‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,8 @@ exports.post = async function(ctx) {
5555

5656
ctx.body = {ok: true};
5757

58-
// don't wait for it! (async)
5958
await updateRepo(ctx.request.body.repository.name);
6059

61-
await Stats.instance().count(ctx.request.body.repository.name);
60+
await Stats.instance().gather(ctx.request.body.repository.name);
6261

63-
};
62+
};

‎handlers/stats.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,14 @@ exports.get = async function (ctx) {
3131
let repoName, repo;
3232

3333
debug('lANG', lang);
34-
34+
3535
for ([repoName, repo] of Object.entries(repos)) {
3636
if (repo.lang === lang) break;
3737
}
3838

39-
const stats = Stats.instance().get(repoName);
39+
const stats = Stats.instance().get(repoName).translation;
4040
const { progress, translated } = stats;
41-
41+
4242
result[lang] = { progress, translated };
4343
});
4444

‎handlers/statsInSvg.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ exports.get = async function(ctx) {
2020
if (repo.lang === lang) break;
2121
}
2222

23-
const { progress } = Stats.instance().get(repoName);
23+
const { progress } = Stats.instance().get(repoName).translation;
2424

2525
ctx.type = 'image/svg+xml';
2626
// ctx.set('cache-control', 'public, max-age=300');

‎lib/countAuthors.js‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
const fs = require('mz/fs');
2+
const path = require('path');
3+
const exec = require('mz/child_process').exec;
4+
const run = require('./run');
5+
const debug = require("debug")('lib:countAuthors');
6+
7+
const countFileAuthorStats = require('./countFileAuthorStats');
8+
const fetchGithubUserByQuery = require('./fetchGithubUserByQuery');
9+
const githubEmailMap = require('../githubEmailMap.json');
10+
11+
async function countAuthors(repoPath, contributorsCache) {
12+
13+
if (contributorsCache.commit) {
14+
let diff = await run(`git diff --name-only ${contributorsCache.commit} HEAD`, {
15+
cwd: repoPath
16+
});
17+
let changedFiles = diff.trim().split('\n');
18+
for(let file of changedFiles) {
19+
delete contributorsCache.files[file];
20+
}
21+
} else {
22+
contributorsCache.files = Object.create(null);
23+
}
24+
25+
26+
let stdout = await run('git grep -Il ""', { // ls-files returns binary files, this doesn't
27+
cwd: repoPath
28+
});
29+
30+
let files = stdout.split(/\n/).map(l => l.trim()).filter(Boolean);
31+
32+
for (let i = 0; i < files.length; i++) {
33+
let file = files[i];
34+
35+
if (file[0] === '.' || file.includes('/.')) {
36+
delete contributorsCache.files[file];
37+
continue; // system file, ignore it
38+
}
39+
40+
if (file.includes('.view')) {
41+
// ignore big test files from view
42+
let fileStat = await fs.stat(path.resolve(repoPath, file));
43+
if (fileStat.size > 60e3) {
44+
delete contributorsCache.files[file];
45+
continue;
46+
}
47+
}
48+
49+
debug(file, i, files.length);
50+
51+
if (!contributorsCache.files[file]) {
52+
contributorsCache.files[file] = await countFileAuthorStats(repoPath, file);
53+
}
54+
55+
}
56+
57+
debug("stats", contributorsCache.files);
58+
59+
contributorsCache.commit = (await run(`git rev-parse --short HEAD`, {
60+
cwd: repoPath
61+
})).trim(); // output ends with \n
62+
63+
let statsByAuthor = Object.create(null);
64+
65+
for(let file in contributorsCache.files) {
66+
let fileStats = contributorsCache.files[file];
67+
for (let author in fileStats) {
68+
statsByAuthor[author] = (statsByAuthor[author] || 0) + fileStats[author];
69+
}
70+
}
71+
72+
debug("stats by author", statsByAuthor);
73+
74+
// build email cache
75+
76+
let emailToGithubUser = contributorsCache.emailToGithubUser || Object.create(null);
77+
78+
for (let githubEmail in statsByAuthor) {
79+
let linesCount = statsByAuthor[githubEmail];
80+
81+
if (linesCount < 10) {
82+
// skip contributors who have less than 10 lines
83+
continue;
84+
}
85+
86+
let githubUser = emailToGithubUser[githubEmail];
87+
88+
89+
// console.log("GITHUBUSER", githubUser);
90+
// console.log("EMAILTOGITHUBUSER", emailToGithubUser);
91+
if (!(githubEmail in emailToGithubUser)) { // may be empty
92+
93+
if (githubEmailMap[githubEmail]) {
94+
let login = githubEmailMap[githubEmail];
95+
githubUser = await fetchGithubUserByQuery(`${login} in:login`);
96+
} else if (githubEmail.endsWith('@users.noreply.github.com')) {
97+
let login = githubEmail.slice(0, -'@users.noreply.github.com'.length);
98+
githubUser = await fetchGithubUserByQuery(`${login} in:login`);
99+
} else {
100+
githubUser = await fetchGithubUserByQuery(`${githubEmail} in:email`);
101+
}
102+
103+
// couldn't find github user =(
104+
if (!githubUser) {
105+
debug("Error: No github user for " + githubEmail);
106+
}
107+
108+
emailToGithubUser[githubEmail] = githubUser || null;
109+
}
110+
}
111+
112+
contributorsCache.emailToGithubUser = emailToGithubUser;
113+
114+
debug(contributorsCache);
115+
116+
}
117+
118+
module.exports = countAuthors;
119+
120+
//countAuthors('/js/javascript-tutorial-zh', {});

‎lib/countFileAuthorStats.js‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
const run = require('./run');
2+
const assert = require('assert');
3+
const debug = require('debug')('lib/countFileAuthorStats');
4+
5+
function escapeShell(cmd) {
6+
return '"'+cmd.replace(/(["\s'$`\\])/g,'\\$1')+'"';
7+
}
8+
9+
async function countFileAuthorStats(repoPath, filePath) {
10+
let stdout = await run('git blame -w --porcelain '+ escapeShell(filePath), {
11+
cwd: repoPath
12+
});
13+
14+
// don't trim! last line may be empty source line
15+
let lines = stdout.split(/\n/);
16+
17+
/*
18+
Each commit is porcelain:
19+
hash [line at the moment of commit] [line now] [number of modified subsequent lines in current blame group]
20+
headerName headerValue
21+
...
22+
TAB sourceLine
23+
24+
if commit is repeated, only hash, no headers
25+
*/
26+
27+
let commits = {};
28+
29+
for (let i = 0; i < lines.length - 1; i++) { // last line of output is always \n
30+
31+
debug(lines.length, i, lines[i]);
32+
33+
let [hash, /*sourceLine*/, /*originalLine*/, /*linesCount*/] = lines[i].split(' ');
34+
35+
debug(hash, "exists", commits[hash]);
36+
37+
if (!commits[hash]) {
38+
i++;
39+
let authorEmail;
40+
41+
while(lines[i][0] !== '\t') { // source line starts prefixed by tab
42+
if (lines[i].startsWith('author-mail ')) {
43+
authorEmail = lines[i].slice('author-mail <'.length, -1);
44+
}
45+
i++;
46+
}
47+
48+
let sourceLine = lines[i].slice(1).trim();
49+
50+
debug(authorEmail, sourceLine, Boolean(sourceLine), '<-- first time');
51+
52+
53+
// must save commit info even for empty lines
54+
// next lines from same commit may be non-empty, but there will be no commit info
55+
commits[hash] = {
56+
linesCount: sourceLine ? 1 : 0, // ignore blank lines
57+
authorEmail
58+
};
59+
60+
} else {
61+
i++;
62+
let sourceLine = lines[i].trim();
63+
debug(lines[i], Boolean(sourceLine), '<-- seen this');
64+
if (!sourceLine) continue; // ignore blank lines
65+
66+
commits[hash].linesCount++;
67+
}
68+
69+
assert(commits[hash].authorEmail);
70+
71+
// some commits may have no lines yet
72+
}
73+
74+
75+
76+
let stats = Object.create(null);
77+
for (let hash in commits) {
78+
let commit = commits[hash];
79+
if (!commit.linesCount) continue; // ignore commit with blank lines only
80+
stats[commit.authorEmail] = (stats[commit.authorEmail] || 0) + commit.linesCount;
81+
}
82+
83+
debug(stats);
84+
return stats;
85+
}
86+
87+
module.exports = countFileAuthorStats;
88+
89+
// countFileAuthorStats('/js/javascript-tutorial-zh', '4-frames-and-windows/01-popup-windows/article.md');

0 commit comments

Comments
 (0)