Skip to content

Commit 7ff8090

Browse files
bojiangbojiang
committed
chore: sort model versions
1 parent ef23282 commit 7ff8090

2 files changed

Lines changed: 157 additions & 123 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ https://github.com/bentoml/OpenLLM/assets/5886138/2aa5f9e4-859c-4be8-91b3-91d8a8
4141

4242
OpenLLM supports a variety of state-of-the-art LLMs. Here are some of the models supported by OpenLLM, each listed with a commonly used model size.
4343

44-
| Model | Parameters | Quantinize | Required VRAM | Start a Server |
44+
| Model | Parameters | Quantinize | Required GPU | Start a Server |
4545
| ------- | ---------- | ---------- | ------------- | ------------------------------- |
4646
| Llama 3 | 8B | - | 24G | `openllm serve llama3:8b` |
4747
| Llama 3 | 8B | AWQ 4bit | 12G | `openllm serve llama3:8b-4bit` |

‎src/openllm/model.py‎

Lines changed: 156 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import re
12
import typing
23
from typing import Optional
34

@@ -9,139 +10,172 @@
910
from openllm.common import FORCE, VERBOSE_LEVEL, BentoInfo, load_config, output
1011
from openllm.repo import ensure_repo_updated, parse_repo_url
1112

12-
app = OpenLLMTyper(help='manage models')
13+
app = OpenLLMTyper(help="manage models")
1314

1415

15-
@app.command(help='get model')
16+
@app.command(help="get model")
1617
def get(tag: str, repo: Optional[str] = None, verbose: bool = False):
17-
if verbose:
18-
VERBOSE_LEVEL.set(20)
19-
bento_info = ensure_bento(tag, repo_name=repo)
20-
if bento_info:
21-
output(bento_info)
22-
23-
24-
@app.command(name='list', help='list available models')
25-
def list_model(tag: Optional[str] = None, repo: Optional[str] = None, verbose: bool = False):
26-
if verbose:
27-
VERBOSE_LEVEL.set(20)
28-
29-
bentos = list_bento(tag=tag, repo_name=repo)
30-
bentos.sort(key=lambda x: x.name)
31-
32-
seen = set()
33-
34-
def is_seen(value):
35-
if value in seen:
36-
return True
37-
seen.add(value)
38-
return False
39-
40-
table = tabulate.tabulate(
41-
[
42-
[
43-
'' if is_seen(bento.name) else bento.name,
44-
bento.tag,
45-
bento.repo.name,
46-
bento.pretty_gpu,
47-
','.join(bento.platforms),
48-
]
49-
for bento in bentos
50-
],
51-
headers=['model', 'version', 'repo', 'required VRAM', 'platforms'],
52-
)
53-
output(table)
54-
55-
56-
def ensure_bento(model: str, target: Optional[DeploymentTarget] = None, repo_name: Optional[str] = None) -> BentoInfo:
57-
bentos = list_bento(model, repo_name=repo_name)
58-
if len(bentos) == 0:
59-
output(f'No model found for {model}', style='red')
60-
raise typer.Exit(1)
61-
62-
if len(bentos) == 1:
63-
if FORCE.get():
64-
output(f'Found model {bentos[0]}', style='green')
65-
return bentos[0]
18+
if verbose:
19+
VERBOSE_LEVEL.set(20)
20+
bento_info = ensure_bento(tag, repo_name=repo)
21+
if bento_info:
22+
output(bento_info)
23+
24+
25+
@app.command(name="list", help="list available models")
26+
def list_model(
27+
tag: Optional[str] = None, repo: Optional[str] = None, verbose: bool = False
28+
):
29+
if verbose:
30+
VERBOSE_LEVEL.set(20)
31+
32+
bentos = list_bento(tag=tag, repo_name=repo)
33+
bentos.sort(key=lambda x: x.name)
34+
35+
seen = set()
36+
37+
def is_seen(value):
38+
if value in seen:
39+
return True
40+
seen.add(value)
41+
return False
42+
43+
table = tabulate.tabulate(
44+
[
45+
[
46+
"" if is_seen(bento.name) else bento.name,
47+
bento.tag,
48+
bento.repo.name,
49+
bento.pretty_gpu,
50+
",".join(bento.platforms),
51+
]
52+
for bento in bentos
53+
],
54+
headers=["model", "version", "repo", "required GPU RAM", "platforms"],
55+
)
56+
output(table)
57+
58+
59+
def ensure_bento(
60+
model: str,
61+
target: Optional[DeploymentTarget] = None,
62+
repo_name: Optional[str] = None,
63+
) -> BentoInfo:
64+
bentos = list_bento(model, repo_name=repo_name)
65+
if len(bentos) == 0:
66+
output(f"No model found for {model}", style="red")
67+
raise typer.Exit(1)
68+
69+
if len(bentos) == 1:
70+
if FORCE.get():
71+
output(f"Found model {bentos[0]}", style="green")
72+
return bentos[0]
73+
if target is None:
74+
return bentos[0]
75+
if can_run(bentos[0], target) <= 0:
76+
return bentos[0]
77+
output(f"Found model {bentos[0]}", style="green")
78+
return bentos[0]
79+
6680
if target is None:
67-
return bentos[0]
68-
if can_run(bentos[0], target) <= 0:
69-
return bentos[0]
70-
output(f'Found model {bentos[0]}', style='green')
71-
return bentos[0]
81+
output(
82+
f"Multiple models match {model}, did you mean one of these?", style="red"
83+
)
84+
for bento in bentos:
85+
output(f" {bento}")
86+
raise typer.Exit(1)
87+
88+
filtered = [bento for bento in bentos if can_run(bento, target) > 0]
89+
if len(filtered) == 0:
90+
output(f"No deployment target found for {model}", style="red")
91+
raise typer.Exit(1)
92+
93+
if len(filtered) == 0:
94+
output(f"No deployment target found for {model}", style="red")
95+
raise typer.Exit(1)
96+
97+
if len(bentos) > 1:
98+
output(
99+
f"Multiple models match {model}, did you mean one of these?", style="red"
100+
)
101+
for bento in bentos:
102+
output(f" {bento}")
103+
raise typer.Exit(1)
72104

73-
if target is None:
74-
output(f'Multiple models match {model}, did you mean one of these?', style='red')
75-
for bento in bentos:
76-
output(f' {bento}')
77-
raise typer.Exit(1)
105+
return bentos[0]
78106

79-
filtered = [bento for bento in bentos if can_run(bento, target) > 0]
80-
if len(filtered) == 0:
81-
output(f'No deployment target found for {model}', style='red')
82-
raise typer.Exit(1)
83107

84-
if len(filtered) == 0:
85-
output(f'No deployment target found for {model}', style='red')
86-
raise typer.Exit(1)
108+
NUMBER_RE = re.compile(r"\d+")
87109

88-
if len(bentos) > 1:
89-
output(f'Multiple models match {model}, did you mean one of these?', style='red')
90-
for bento in bentos:
91-
output(f' {bento}')
92-
raise typer.Exit(1)
93110

94-
return bentos[0]
111+
def _extract_first_number(s: str):
112+
match = NUMBER_RE.search(s)
113+
if match:
114+
return int(match.group())
115+
else:
116+
return 100
95117

96118

97119
def list_bento(
98-
tag: typing.Optional[str] = None, repo_name: typing.Optional[str] = None, include_alias: bool = False
120+
tag: typing.Optional[str] = None,
121+
repo_name: typing.Optional[str] = None,
122+
include_alias: bool = False,
99123
) -> typing.List[BentoInfo]:
100-
ensure_repo_updated()
101-
102-
if repo_name is not None:
124+
ensure_repo_updated()
125+
126+
if repo_name is not None:
127+
config = load_config()
128+
if repo_name not in config.repos:
129+
output(f"Repo `{repo_name}` not found, did you mean one of these?")
130+
for repo_name in config.repos:
131+
output(f" {repo_name}")
132+
raise typer.Exit(1)
133+
134+
if not tag:
135+
glob_pattern = "bentoml/bentos/*/*"
136+
elif ":" in tag:
137+
bento_name, version = tag.split(":")
138+
glob_pattern = f"bentoml/bentos/{bento_name}/{version}"
139+
else:
140+
glob_pattern = f"bentoml/bentos/{tag}/*"
141+
142+
model_list = []
103143
config = load_config()
104-
if repo_name not in config.repos:
105-
output(f'Repo `{repo_name}` not found, did you mean one of these?')
106-
for repo_name in config.repos:
107-
output(f' {repo_name}')
108-
raise typer.Exit(1)
109-
110-
if not tag:
111-
glob_pattern = 'bentoml/bentos/*/*'
112-
elif ':' in tag:
113-
bento_name, version = tag.split(':')
114-
glob_pattern = f'bentoml/bentos/{bento_name}/{version}'
115-
else:
116-
glob_pattern = f'bentoml/bentos/{tag}/*'
117-
118-
model_list = []
119-
config = load_config()
120-
for _repo_name, repo_url in config.repos.items():
121-
if repo_name is not None and _repo_name != repo_name:
122-
continue
123-
repo = parse_repo_url(repo_url, _repo_name)
124-
for path in repo.path.glob(glob_pattern):
125-
if path.is_dir() and (path / 'bento.yaml').exists():
126-
model = BentoInfo(repo=repo, path=path)
127-
elif path.is_file():
128-
with open(path) as f:
129-
origin_name = f.read().strip()
130-
origin_path = path.parent / origin_name
131-
model = BentoInfo(alias=path.name, repo=repo, path=origin_path)
132-
else:
133-
model = None
134-
if model:
135-
model_list.append(model)
136-
model_list.sort(key=lambda x: x.tag)
137-
if not include_alias:
138-
seen = set()
139-
model_list = [
140-
x
141-
for x in model_list
142-
if not (
143-
f"{x.bento_yaml['name']}:{x.bento_yaml['version']}" in seen
144-
or seen.add(f"{x.bento_yaml['name']}:{x.bento_yaml['version']}")
145-
)
146-
]
147-
return model_list
144+
for _repo_name, repo_url in config.repos.items():
145+
if repo_name is not None and _repo_name != repo_name:
146+
continue
147+
repo = parse_repo_url(repo_url, _repo_name)
148+
149+
paths = sorted(
150+
repo.path.glob(glob_pattern),
151+
key=lambda x: (
152+
x.parent.name,
153+
_extract_first_number(x.name),
154+
len(x.name),
155+
x.name,
156+
),
157+
)
158+
for path in paths:
159+
if path.is_dir() and (path / "bento.yaml").exists():
160+
model = BentoInfo(repo=repo, path=path)
161+
elif path.is_file():
162+
with open(path) as f:
163+
origin_name = f.read().strip()
164+
origin_path = path.parent / origin_name
165+
model = BentoInfo(alias=path.name, repo=repo, path=origin_path)
166+
else:
167+
model = None
168+
if model:
169+
model_list.append(model)
170+
171+
if not include_alias:
172+
seen = set()
173+
model_list = [
174+
x
175+
for x in model_list
176+
if not (
177+
f"{x.bento_yaml['name']}:{x.bento_yaml['version']}" in seen
178+
or seen.add(f"{x.bento_yaml['name']}:{x.bento_yaml['version']}")
179+
)
180+
]
181+
return model_list

0 commit comments

Comments
 (0)