Skip to content

Commit 7135cce

Browse files
authored
Merge pull request #2 from lm-sys/main
merge
2 parents 0e8e22f + f2e6ca9 commit 7135cce

File tree

13 files changed

+470
-154
lines changed

13 files changed

+470
-154
lines changed

fastchat/conversation.py

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,6 +1048,23 @@ def get_conv_template(name: str) -> Conversation:
10481048
)
10491049
)
10501050

1051+
register_conv_template(
1052+
Conversation(
1053+
name="gemini-dev",
1054+
roles=("user", "model"),
1055+
sep_style=SeparatorStyle.DEFAULT,
1056+
sep=None,
1057+
system_message=(
1058+
"You are a friendly and helpful assistant.\n"
1059+
"Ensure your answers are complete, unless the user requests a more concise approach.\n"
1060+
"When generating code, offer explanations for code segments as necessary and maintain good coding practices.\n"
1061+
"When presented with inquiries seeking information, provide answers that reflect a deep understanding of the field, guaranteeing their correctness.\n"
1062+
"For any non-english queries, respond in the same language as the prompt unless otherwise specified by the user.\n"
1063+
"For prompts involving reasoning, provide a clear explanation of each step in the reasoning process before presenting the final answer."
1064+
),
1065+
)
1066+
)
1067+
10511068
# BiLLa default template
10521069
register_conv_template(
10531070
Conversation(
@@ -1789,16 +1806,6 @@ def get_conv_template(name: str) -> Conversation:
17891806
)
17901807
)
17911808

1792-
register_conv_template(
1793-
Conversation(
1794-
name="reka",
1795-
system_message="",
1796-
roles=("user", "assistant"),
1797-
sep_style=None,
1798-
sep=None,
1799-
)
1800-
)
1801-
18021809

18031810
if __name__ == "__main__":
18041811
from fastchat.conversation import get_conv_template

fastchat/model/model_adapter.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1201,6 +1201,19 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
12011201
return get_conv_template("gemini")
12021202

12031203

1204+
class GeminiDevAdapter(BaseModelAdapter):
1205+
"""The model adapter for Gemini 1.5 Pro"""
1206+
1207+
def match(self, model_path: str):
1208+
return "gemini-1.5-pro" in model_path.lower()
1209+
1210+
def load_model(self, model_path: str, from_pretrained_kwargs: dict):
1211+
raise NotImplementedError()
1212+
1213+
def get_default_conv_template(self, model_path: str) -> Conversation:
1214+
return get_conv_template("gemini-dev")
1215+
1216+
12041217
class BiLLaAdapter(BaseModelAdapter):
12051218
"""The model adapter for Neutralzz/BiLLa-7B-SFT"""
12061219

@@ -2403,7 +2416,7 @@ def match(self, model_path: str):
24032416
return "reka" in model_path.lower()
24042417

24052418
def get_default_conv_template(self, model_path: str) -> Conversation:
2406-
return get_conv_template("reka")
2419+
return get_conv_template("api_based_default")
24072420

24082421

24092422
# Note: the registration order matters.
@@ -2431,6 +2444,7 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
24312444
register_model_adapter(BardAdapter)
24322445
register_model_adapter(PaLM2Adapter)
24332446
register_model_adapter(GeminiAdapter)
2447+
register_model_adapter(GeminiDevAdapter)
24342448
register_model_adapter(GemmaAdapter)
24352449
register_model_adapter(ChatGPTAdapter)
24362450
register_model_adapter(AzureOpenAIAdapter)

fastchat/model/model_registry.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@ def get_model_info(name: str) -> ModelInfo:
5454
"Claude by Anthropic",
5555
)
5656

57+
register_model_info(
58+
["reka-flash", "reka-flash-online"],
59+
"Reka Flash",
60+
"https://www.reka.ai/news/reka-flash-efficient-and-capable-multimodal-language-models",
61+
"Multimodal model by Reka",
62+
)
63+
5764
register_model_info(
5865
["command-r-plus"],
5966
"Command-R-Plus",
@@ -743,10 +750,3 @@ def get_model_info(name: str) -> ModelInfo:
743750
"https://huggingface.co/cllm",
744751
"consistency-llm is a new generation of parallel decoder LLMs with fast generation speed.",
745752
)
746-
747-
register_model_info(
748-
["reka-flash"],
749-
"Reka Flash",
750-
"https://reka.ai/reka-flash",
751-
"Multimodal model by Reka",
752-
)

fastchat/serve/api_provider.py

Lines changed: 37 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,10 @@ def get_api_provider_stream_iter(
6464
vertex_ai=True,
6565
)
6666
elif model_api_dict["api_type"] == "gemini":
67+
prompt = conv.to_openai_api_messages()
6768
stream_iter = gemini_api_stream_iter(
6869
model_api_dict["model_name"],
69-
conv,
70+
prompt,
7071
temperature,
7172
top_p,
7273
max_new_tokens,
@@ -475,10 +476,16 @@ def anthropic_message_api_stream_iter(
475476

476477

477478
def gemini_api_stream_iter(
478-
model_name, conv, temperature, top_p, max_new_tokens, api_key=None
479+
model_name, messages, temperature, top_p, max_new_tokens, api_key=None
479480
):
480481
import google.generativeai as genai # pip install google-generativeai
481482

483+
OPENAI_TO_GEMINI_ROLE_MAP = {
484+
"user": "user",
485+
"assistant": "model",
486+
"system": "system",
487+
}
488+
482489
if api_key is None:
483490
api_key = os.environ["GEMINI_API_KEY"]
484491
genai.configure(api_key=api_key)
@@ -490,7 +497,7 @@ def gemini_api_stream_iter(
490497
}
491498
params = {
492499
"model": model_name,
493-
"prompt": conv,
500+
"prompt": messages,
494501
}
495502
params.update(generation_config)
496503
logger.info(f"==== request ====\n{params}")
@@ -501,16 +508,24 @@ def gemini_api_stream_iter(
501508
{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
502509
{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
503510
]
511+
512+
history = []
513+
system_prompt = None
514+
for message in messages[:-1]:
515+
role = OPENAI_TO_GEMINI_ROLE_MAP[message["role"]]
516+
if role == "system":
517+
system_prompt = message["content"]
518+
continue
519+
history.append({"role": role, "parts": message["content"]})
520+
504521
model = genai.GenerativeModel(
505522
model_name=model_name,
523+
system_instruction=system_prompt,
506524
generation_config=generation_config,
507525
safety_settings=safety_settings,
508526
)
509-
history = []
510-
for role, message in conv.messages[:-2]:
511-
history.append({"role": role, "parts": message})
512527
convo = model.start_chat(history=history)
513-
response = convo.send_message(conv.messages[-2][1], stream=True)
528+
response = convo.send_message(messages[-1]["content"], stream=True)
514529

515530
try:
516531
text = ""
@@ -893,14 +908,21 @@ def reka_api_stream_iter(
893908
# merge consecutive rounds with same role into one round
894909
chat_history[-1]["text"] += "\n\n" + message["content"]
895910

896-
request = dict(
897-
model_name=model_name,
898-
conversation_history=chat_history,
899-
temperature=temperature,
900-
request_output_len=max_new_tokens,
901-
runtime_top_p=top_p,
902-
stream=True,
903-
)
911+
use_search_engine = False
912+
if "-online" in model_name:
913+
model_name = model_name.replace("-online", "")
914+
use_search_engine = True
915+
request = {
916+
"model_name": model_name,
917+
"conversation_history": chat_history,
918+
"temperature": temperature,
919+
"request_output_len": max_new_tokens,
920+
"runtime_top_p": top_p,
921+
"stream": True,
922+
"use_search_engine": use_search_engine,
923+
}
924+
logger.info(f"==== request ====\n{request}")
925+
904926
response = requests.post(
905927
api_base,
906928
stream=True,

fastchat/serve/gradio_block_arena_anony.py

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -182,21 +182,22 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
182182
"dbrx-instruct": 1,
183183
"command-r-plus": 4,
184184
"command-r": 2,
185-
"gemini-pro-dev-api": 2,
185+
"reka-flash": 4,
186+
"reka-flash-online": 4,
186187
"qwen1.5-72b-chat": 2,
187188
"qwen1.5-32b-chat": 2,
188189
"qwen1.5-14b-chat": 2,
189190
"qwen1.5-7b-chat": 2,
190191
"gemma-1.1-7b-it": 2,
191-
"gemma-1.1-2b-it": 2,
192+
"gemma-1.1-2b-it": 1,
192193
"mixtral-8x7b-instruct-v0.1": 4,
193194
"mistral-7b-instruct-v0.2": 2,
194195
"mistral-large-2402": 4,
195196
"mistral-medium": 2,
196197
"starling-lm-7b-beta": 2,
197198
# tier 1
198-
"deluxe-chat-v1.3": 3,
199-
"llama-2-70b-chat": 4,
199+
"deluxe-chat-v1.3": 2,
200+
"llama-2-70b-chat": 2,
200201
"llama-2-13b-chat": 1,
201202
"llama-2-7b-chat": 1,
202203
"vicuna-33b": 1,
@@ -210,33 +211,32 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
210211
"gpt-4-1106-preview",
211212
"gpt-4-0125-preview",
212213
"claude-3-opus-20240229",
213-
"claude-3-sonnet-20240229",
214-
"mistral-large-2402",
214+
"gemini-pro-dev-api",
215215
},
216-
"command-r-plus": {
217-
"command-r",
218-
"mixtral-8x7b-instruct-v0.1",
219-
"gpt-4-0125-preview",
220-
"gpt-4-1106-preview",
216+
"gemini-pro-dev-api": {
217+
"gpt-4-turbo-2024-04-09",
221218
"claude-3-opus-20240229",
219+
"gpt-4-0125-preview",
222220
"claude-3-sonnet-20240229",
221+
},
222+
"reka-flash": {
223+
"qwen1.5-72b-chat",
224+
"claude-3-haiku-20240307",
225+
"command-r-plus",
226+
"command-r",
227+
},
228+
"reka-flash-online": {
229+
"qwen1.5-72b-chat",
223230
"claude-3-haiku-20240307",
231+
"command-r-plus",
232+
"command-r",
224233
},
225234
"deluxe-chat-v1.3": {
226235
"gpt-4-1106-preview",
227236
"gpt-4-0125-preview",
228237
"claude-3-opus-20240229",
229238
"claude-3-sonnet-20240229",
230239
},
231-
"qwen1.5-72b-chat": {
232-
"gpt-3.5-turbo-0125",
233-
"gpt-4-0613",
234-
"gpt-4-0125-preview",
235-
"llama-2-70b-chat",
236-
"mixtral-8x7b-instruct-v0.1",
237-
"mistral-medium",
238-
"yi-34b-chat",
239-
},
240240
"qwen1.5-32b-chat": {
241241
"gpt-3.5-turbo-0125",
242242
"gpt-4-0613",
@@ -261,13 +261,6 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
261261
"mistral-next",
262262
"claude-3-sonnet-20240229",
263263
},
264-
"gemma-1.1-7b-it": {
265-
"gpt-3.5-turbo-0125",
266-
"mixtral-8x7b-instruct-v0.1",
267-
"starling-lm-7b-beta",
268-
"llama-2-7b-chat",
269-
"mistral-7b-instruct-v0.2",
270-
},
271264
"gemma-1.1-2b-it": {
272265
"gpt-3.5-turbo-0125",
273266
"mixtral-8x7b-instruct-v0.1",
@@ -280,6 +273,7 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
280273
"qwen1.5-72b-chat",
281274
"mistral-large-2402",
282275
"command-r-plus",
276+
"claude-3-haiku-20240307",
283277
},
284278
}
285279

fastchat/serve/gradio_web_server.py

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,15 @@
6666
The service collects user dialogue data, including both text and images, and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license.
6767
6868
### Acknowledgment
69-
We thank [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).
69+
We thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).
7070
7171
<div class="sponsor-image-about">
72+
<img src="https://storage.googleapis.com/public-arena-asset/skylab.png" alt="SkyLab">
7273
<img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
7374
<img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
7475
<img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
7576
<img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
77+
<img src="https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png" alt="Hyperbolic">
7678
<img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
7779
<img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
7880
</div>
@@ -490,6 +492,9 @@ def bot_response(
490492
if recommended_config is not None:
491493
temperature = recommended_config.get("temperature", temperature)
492494
top_p = recommended_config.get("top_p", top_p)
495+
max_new_tokens = recommended_config.get(
496+
"max_new_tokens", max_new_tokens
497+
)
493498

494499
stream_iter = get_api_provider_stream_iter(
495500
conv,
@@ -617,10 +622,10 @@ def bot_response(
617622
padding-bottom: 6px;
618623
}
619624
#arena_leaderboard_dataframe table {
620-
font-size: 115%;
625+
font-size: 110%;
621626
}
622627
#full_leaderboard_dataframe table {
623-
font-size: 115%;
628+
font-size: 110%;
624629
}
625630
#model_description_markdown {
626631
font-size: 110% !important;
@@ -644,9 +649,6 @@ def bot_response(
644649
#chatbot .prose {
645650
font-size: 105% !important;
646651
}
647-
footer {
648-
display:none !important;
649-
}
650652
.sponsor-image-about img {
651653
margin: 0 20px;
652654
margin-top: 20px;
@@ -750,13 +752,15 @@ def build_about():
750752
751753
## Acknowledgment
752754
We thank [SkyPilot](https://github.com/skypilot-org/skypilot) and [Gradio](https://github.com/gradio-app/gradio) team for their system support.
753-
We also thank [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous sponsorship. Learn more about partnership [here](https://lmsys.org/donations/).
755+
We also thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous sponsorship. Learn more about partnership [here](https://lmsys.org/donations/).
754756
755757
<div class="sponsor-image-about">
758+
<img src="https://storage.googleapis.com/public-arena-asset/skylab.png" alt="SkyLab">
756759
<img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
757760
<img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
758761
<img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
759762
<img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
763+
<img src="https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png" alt="Hyperbolic">
760764
<img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
761765
<img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
762766
</div>

fastchat/serve/gradio_web_server_multi.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,4 +287,5 @@ def build_demo(models, vl_models, elo_results_file, leaderboard_table_file):
287287
max_threads=200,
288288
auth=auth,
289289
root_path=args.gradio_root_path,
290+
show_api=False,
290291
)

fastchat/serve/monitor/clean_chat_data.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ def clean_chat_data(log_files, action_type):
8989
if not isinstance(model, str):
9090
ct_invalid += 1
9191
continue
92-
model = replace_model_name(model)
92+
model = replace_model_name(model, row["tstamp"])
9393

9494
try:
9595
lang_code = detect_language(state["messages"][state["offset"]][1])

0 commit comments

Comments
 (0)