Skip to content

Commit c1f9df2

Browse files
hoangsvithoangsvit
committed
Update location page generation script
1 parent 022ec8a commit c1f9df2

1 file changed

Lines changed: 61 additions & 61 deletions

File tree

‎.github/workflows/daily_update.yml‎

Lines changed: 61 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -31,72 +31,72 @@ jobs:
3131
run: |
3232
import csv, datetime, glob, os, re, subprocess, sys
3333
34-
MAX_IN_RUN = 4
35-
STALE_DAYS = 1
36-
GENERATED = {
37-
"%s.md": "---\ntype: location\nlocation: %s\nmode: commits\n---\n",
38-
"%s_private.md": "---\ntype: location\nlocation: %s\nmode: all\n---\n",
39-
"%s_public.md": "---\ntype: location\nlocation: %s\nmode: contributions\n---\n",
40-
"rank_only/%s.json": "---\nlayout: rank_only\nlocation: %s\n---\n",
41-
}
34+
MAX_IN_RUN = 3
35+
STALE_DAYS = 2
36+
GENERATED = {
37+
"%s.md": "---\ntype: location\nlocation: %s\nmode: commits\n---\n",
38+
"%s_private.md": "---\ntype: location\nlocation: %s\nmode: all\n---\n",
39+
"%s_public.md": "---\ntype: location\nlocation: %s\nmode: contributions\n---\n",
40+
"rank_only/%s.json": "---\nlayout: rank_only\nlocation: %s\n---\n",
41+
}
4242
43-
subprocess.run("mkdir -p rank_only", shell=True, check=True)
44-
preset_data = subprocess.run(["most-active-github-users-counter", "--list-presets"], capture_output=True, text=True).stdout
45-
all_presets = dict([row.pop('preset'), row] for row in csv.DictReader(preset_data.split('\n')))
46-
regenerated = False
47-
to_process = []
43+
subprocess.run("mkdir -p rank_only", shell=True, check=True)
44+
preset_data = subprocess.run(["most-active-github-users-counter", "--list-presets"], capture_output=True, text=True).stdout
45+
all_presets = dict([row.pop('preset'), row] for row in csv.DictReader(preset_data.split('\n')))
46+
regenerated = False
47+
to_process = []
4848
49-
# generate markdown files for each preset
50-
expected = ["index.md", "README.md"] # shouldn't be deleted!
51-
flookup = {}
52-
for preset in sorted(all_presets.keys()):
53-
filename = preset.replace(" ", "_")
54-
flookup[filename] = preset
55-
expected.append("_data/locations/%s.yml" % filename)
56-
if not os.path.exists(expected[-1]):
57-
to_process.append(filename)
58-
else:
59-
match = re.search("^definition_checksum: (\w+)$", open(expected[-1]).read(), re.M)
60-
if not match or match.group(1) != all_presets[preset]['definition_checksum']:
61-
to_process.append(filename)
62-
for name, content in GENERATED.items():
63-
expected.append(name % filename)
64-
if not os.path.exists(expected[-1]):
65-
regenerated = True
66-
with open(expected[-1], "w") as f:
67-
f.write(content % filename)
49+
# generate markdown files for each preset
50+
expected = ["index.md", "README.md"] # shouldn't be deleted!
51+
flookup = {}
52+
for preset in sorted(all_presets.keys()):
53+
filename = preset.replace(" ", "_")
54+
flookup[filename] = preset
55+
expected.append("_data/locations/%s.yml" % filename)
56+
if not os.path.exists(expected[-1]):
57+
to_process.append(filename)
58+
else:
59+
match = re.search("^definition_checksum: (\w+)$", open(expected[-1]).read(), re.M)
60+
if not match or match.group(1) != all_presets[preset]['definition_checksum']:
61+
to_process.append(filename)
62+
for name, content in GENERATED.items():
63+
expected.append(name % filename)
64+
if not os.path.exists(expected[-1]):
65+
regenerated = True
66+
with open(expected[-1], "w") as f:
67+
f.write(content % filename)
6868
69-
# remove files for presets no longer supported (if any)
70-
for filename in sorted(glob.glob("*.md") + glob.glob("_data/locations/*.yml") + glob.glob("rank_only/*.json")):
71-
if filename not in expected:
72-
os.remove(filename)
73-
regenerated = True
69+
# remove files for presets no longer supported (if any)
70+
for filename in sorted(glob.glob("*.md") + glob.glob("_data/locations/*.yml") + glob.glob("rank_only/*.json")):
71+
if filename not in expected:
72+
os.remove(filename)
73+
regenerated = True
7474
75-
# find if any location data is stale
76-
get_mtime = lambda n: datetime.date.fromisoformat(subprocess.run(["git", "log", "-1", "--pretty=%as", n], capture_output=True, text=True).stdout.strip())
77-
locations = sorted([get_mtime(n), os.path.basename(n)] for n in glob.glob("_data/locations/*.yml"))
78-
today = datetime.date.today()
79-
for mtime, filename in locations:
80-
if (today - mtime).days >= STALE_DAYS:
81-
to_process.append(os.path.splitext(filename)[0])
75+
# find if any location data is stale
76+
get_mtime = lambda n: datetime.date.fromisoformat(subprocess.run(["git", "log", "-1", "--pretty=%as", n], capture_output=True, text=True).stdout.strip())
77+
locations = sorted([get_mtime(n), os.path.basename(n)] for n in glob.glob("_data/locations/*.yml"))
78+
today = datetime.date.today()
79+
for mtime, filename in locations:
80+
if (today - mtime).days >= STALE_DAYS:
81+
to_process.append(os.path.splitext(filename)[0])
8282
83-
if not regenerated and len(to_process) == 0:
84-
sys.exit(0)
83+
if not regenerated and len(to_process) == 0:
84+
sys.exit(0)
8585
86-
if regenerated:
87-
subprocess.run('git add *.md _data/locations/*.yml rank_only/*.json && git commit -am "regenerate location pages"', shell=True, check=True)
86+
if regenerated:
87+
subprocess.run('git add *.md _data/locations/*.yml rank_only/*.json && git commit -am "regenerate location pages"', shell=True, check=True)
8888
89-
for key in to_process[0:MAX_IN_RUN]:
90-
print("Running: %s" % key)
91-
preset = flookup[key]
92-
status = subprocess.run(["most-active-github-users-counter", "--token", os.environ["GITHUB_TOKEN"], "--preset", preset, "--output", "yaml"], capture_output=True, text=True)
93-
if status.returncode == 0:
94-
with open("_data/locations/%s.yml" % key, "w") as f:
95-
f.write("page: %s.html\n%s" % (key, status.stdout))
96-
subprocess.run('git add _data/locations/%s.yml && git commit -m "%s: updates for %s"' % (key, preset, today.isoformat()), shell=True, check=True)
97-
else:
98-
subprocess.run("git push origin gh-pages", shell=True) # push successful commits, if any
99-
print("FAILED with exit code %d\n--- stdout ---\n%s\n--- stderr ---\n%s" % (status.returncode, status.stdout, status.stderr))
100-
sys.exit(1)
89+
for key in to_process[0:MAX_IN_RUN]:
90+
print("Running: %s" % key)
91+
preset = flookup[key]
92+
status = subprocess.run(["most-active-github-users-counter", "--token", os.environ["GITHUB_TOKEN"], "--preset", preset, "--output", "yaml"], capture_output=True, text=True)
93+
if status.returncode == 0:
94+
with open("_data/locations/%s.yml" % key, "w") as f:
95+
f.write("page: %s.html\n%s" % (key, status.stdout))
96+
subprocess.run('git add _data/locations/%s.yml && git commit -m "%s: updates for %s"' % (key, preset, today.isoformat()), shell=True, check=True)
97+
else:
98+
subprocess.run("git push origin gh-pages", shell=True) # push successful commits, if any
99+
print("FAILED with exit code %d\n--- stdout ---\n%s\n--- stderr ---\n%s" % (status.returncode, status.stdout, status.stderr))
100+
sys.exit(1)
101101
102-
subprocess.run("git push origin gh-pages", shell=True, check=True)
102+
subprocess.run("git push origin gh-pages", shell=True, check=True)

0 commit comments

Comments
 (0)