Merge pull request #2 from lm-sys/main

merge

Author: icowan

fastchat/conversation.py

@@ -1048,6 +1048,23 @@ def get_conv_template(name: str) -> Conversation:
     )
 )
 
+register_conv_template(
+    Conversation(
+        name="gemini-dev",
+        roles=("user", "model"),
+        sep_style=SeparatorStyle.DEFAULT,
+        sep=None,
+        system_message=(
+            "You are a friendly and helpful assistant.\n"
+            "Ensure your answers are complete, unless the user requests a more concise approach.\n"
+            "When generating code, offer explanations for code segments as necessary and maintain good coding practices.\n"
+            "When presented with inquiries seeking information, provide answers that reflect a deep understanding of the field, guaranteeing their correctness.\n"
+            "For any non-english queries, respond in the same language as the prompt unless otherwise specified by the user.\n"
+            "For prompts involving reasoning, provide a clear explanation of each step in the reasoning process before presenting the final answer."
+        ),
+    )
+)
+
 # BiLLa default template
 register_conv_template(
     Conversation(
@@ -1789,16 +1806,6 @@ def get_conv_template(name: str) -> Conversation:
     )
 )
 
-register_conv_template(
-    Conversation(
-        name="reka",
-        system_message="",
-        roles=("user", "assistant"),
-        sep_style=None,
-        sep=None,
-    )
-)
-
 
 if __name__ == "__main__":
     from fastchat.conversation import get_conv_template

fastchat/model/model_adapter.py

@@ -1201,6 +1201,19 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
         return get_conv_template("gemini")
 
 
+class GeminiDevAdapter(BaseModelAdapter):
+    """The model adapter for Gemini 1.5 Pro"""
+
+    def match(self, model_path: str):
+        return "gemini-1.5-pro" in model_path.lower()
+
+    def load_model(self, model_path: str, from_pretrained_kwargs: dict):
+        raise NotImplementedError()
+
+    def get_default_conv_template(self, model_path: str) -> Conversation:
+        return get_conv_template("gemini-dev")
+
+
 class BiLLaAdapter(BaseModelAdapter):
     """The model adapter for Neutralzz/BiLLa-7B-SFT"""
 
@@ -2403,7 +2416,7 @@ def match(self, model_path: str):
         return "reka" in model_path.lower()
 
     def get_default_conv_template(self, model_path: str) -> Conversation:
-        return get_conv_template("reka")
+        return get_conv_template("api_based_default")
 
 
 # Note: the registration order matters.
@@ -2431,6 +2444,7 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
 register_model_adapter(BardAdapter)
 register_model_adapter(PaLM2Adapter)
 register_model_adapter(GeminiAdapter)
+register_model_adapter(GeminiDevAdapter)
 register_model_adapter(GemmaAdapter)
 register_model_adapter(ChatGPTAdapter)
 register_model_adapter(AzureOpenAIAdapter)

fastchat/model/model_registry.py

@@ -54,6 +54,13 @@ def get_model_info(name: str) -> ModelInfo:
     "Claude by Anthropic",
 )
 
+register_model_info(
+    ["reka-flash", "reka-flash-online"],
+    "Reka Flash",
+    "https://www.reka.ai/news/reka-flash-efficient-and-capable-multimodal-language-models",
+    "Multimodal model by Reka",
+)
+
 register_model_info(
     ["command-r-plus"],
     "Command-R-Plus",
@@ -743,10 +750,3 @@ def get_model_info(name: str) -> ModelInfo:
     "https://huggingface.co/cllm",
     "consistency-llm is a new generation of parallel decoder LLMs with fast generation speed.",
 )
-
-register_model_info(
-    ["reka-flash"],
-    "Reka Flash",
-    "https://reka.ai/reka-flash",
-    "Multimodal model by Reka",
-)

fastchat/serve/api_provider.py

@@ -64,9 +64,10 @@ def get_api_provider_stream_iter(
             vertex_ai=True,
         )
     elif model_api_dict["api_type"] == "gemini":
+        prompt = conv.to_openai_api_messages()
         stream_iter = gemini_api_stream_iter(
             model_api_dict["model_name"],
-            conv,
+            prompt,
             temperature,
             top_p,
             max_new_tokens,
@@ -475,10 +476,16 @@ def anthropic_message_api_stream_iter(
 
 
 def gemini_api_stream_iter(
-    model_name, conv, temperature, top_p, max_new_tokens, api_key=None
+    model_name, messages, temperature, top_p, max_new_tokens, api_key=None
 ):
     import google.generativeai as genai  # pip install google-generativeai
 
+    OPENAI_TO_GEMINI_ROLE_MAP = {
+        "user": "user",
+        "assistant": "model",
+        "system": "system",
+    }
+
     if api_key is None:
         api_key = os.environ["GEMINI_API_KEY"]
     genai.configure(api_key=api_key)
@@ -490,7 +497,7 @@ def gemini_api_stream_iter(
     }
     params = {
         "model": model_name,
-        "prompt": conv,
+        "prompt": messages,
     }
     params.update(generation_config)
     logger.info(f"==== request ====\n{params}")
@@ -501,16 +508,24 @@ def gemini_api_stream_iter(
         {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
         {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
     ]
+
+    history = []
+    system_prompt = None
+    for message in messages[:-1]:
+        role = OPENAI_TO_GEMINI_ROLE_MAP[message["role"]]
+        if role == "system":
+            system_prompt = message["content"]
+            continue
+        history.append({"role": role, "parts": message["content"]})
+
     model = genai.GenerativeModel(
         model_name=model_name,
+        system_instruction=system_prompt,
         generation_config=generation_config,
         safety_settings=safety_settings,
     )
-    history = []
-    for role, message in conv.messages[:-2]:
-        history.append({"role": role, "parts": message})
     convo = model.start_chat(history=history)
-    response = convo.send_message(conv.messages[-2][1], stream=True)
+    response = convo.send_message(messages[-1]["content"], stream=True)
 
     try:
         text = ""
@@ -893,14 +908,21 @@ def reka_api_stream_iter(
             # merge consecutive rounds with same role into one round
             chat_history[-1]["text"] += "\n\n" + message["content"]
 
-    request = dict(
-        model_name=model_name,
-        conversation_history=chat_history,
-        temperature=temperature,
-        request_output_len=max_new_tokens,
-        runtime_top_p=top_p,
-        stream=True,
-    )
+    use_search_engine = False
+    if "-online" in model_name:
+        model_name = model_name.replace("-online", "")
+        use_search_engine = True
+    request = {
+        "model_name": model_name,
+        "conversation_history": chat_history,
+        "temperature": temperature,
+        "request_output_len": max_new_tokens,
+        "runtime_top_p": top_p,
+        "stream": True,
+        "use_search_engine": use_search_engine,
+    }
+    logger.info(f"==== request ====\n{request}")
+
     response = requests.post(
         api_base,
         stream=True,

fastchat/serve/gradio_block_arena_anony.py

@@ -182,21 +182,22 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
     "dbrx-instruct": 1,
     "command-r-plus": 4,
     "command-r": 2,
-    "gemini-pro-dev-api": 2,
+    "reka-flash": 4,
+    "reka-flash-online": 4,
     "qwen1.5-72b-chat": 2,
     "qwen1.5-32b-chat": 2,
     "qwen1.5-14b-chat": 2,
     "qwen1.5-7b-chat": 2,
     "gemma-1.1-7b-it": 2,
-    "gemma-1.1-2b-it": 2,
+    "gemma-1.1-2b-it": 1,
     "mixtral-8x7b-instruct-v0.1": 4,
     "mistral-7b-instruct-v0.2": 2,
     "mistral-large-2402": 4,
     "mistral-medium": 2,
     "starling-lm-7b-beta": 2,
     # tier 1
-    "deluxe-chat-v1.3": 3,
-    "llama-2-70b-chat": 4,
+    "deluxe-chat-v1.3": 2,
+    "llama-2-70b-chat": 2,
     "llama-2-13b-chat": 1,
     "llama-2-7b-chat": 1,
     "vicuna-33b": 1,
@@ -210,33 +211,32 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
         "gpt-4-1106-preview",
         "gpt-4-0125-preview",
         "claude-3-opus-20240229",
-        "claude-3-sonnet-20240229",
-        "mistral-large-2402",
+        "gemini-pro-dev-api",
     },
-    "command-r-plus": {
-        "command-r",
-        "mixtral-8x7b-instruct-v0.1",
-        "gpt-4-0125-preview",
-        "gpt-4-1106-preview",
+    "gemini-pro-dev-api": {
+        "gpt-4-turbo-2024-04-09",
         "claude-3-opus-20240229",
+        "gpt-4-0125-preview",
         "claude-3-sonnet-20240229",
+    },
+    "reka-flash": {
+        "qwen1.5-72b-chat",
+        "claude-3-haiku-20240307",
+        "command-r-plus",
+        "command-r",
+    },
+    "reka-flash-online": {
+        "qwen1.5-72b-chat",
         "claude-3-haiku-20240307",
+        "command-r-plus",
+        "command-r",
     },
     "deluxe-chat-v1.3": {
         "gpt-4-1106-preview",
         "gpt-4-0125-preview",
         "claude-3-opus-20240229",
         "claude-3-sonnet-20240229",
     },
-    "qwen1.5-72b-chat": {
-        "gpt-3.5-turbo-0125",
-        "gpt-4-0613",
-        "gpt-4-0125-preview",
-        "llama-2-70b-chat",
-        "mixtral-8x7b-instruct-v0.1",
-        "mistral-medium",
-        "yi-34b-chat",
-    },
     "qwen1.5-32b-chat": {
         "gpt-3.5-turbo-0125",
         "gpt-4-0613",
@@ -261,13 +261,6 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
         "mistral-next",
         "claude-3-sonnet-20240229",
     },
-    "gemma-1.1-7b-it": {
-        "gpt-3.5-turbo-0125",
-        "mixtral-8x7b-instruct-v0.1",
-        "starling-lm-7b-beta",
-        "llama-2-7b-chat",
-        "mistral-7b-instruct-v0.2",
-    },
     "gemma-1.1-2b-it": {
         "gpt-3.5-turbo-0125",
         "mixtral-8x7b-instruct-v0.1",
@@ -280,6 +273,7 @@ def share_click(state0, state1, model_selector0, model_selector1, request: gr.Re
         "qwen1.5-72b-chat",
         "mistral-large-2402",
         "command-r-plus",
+        "claude-3-haiku-20240307",
     },
 }
 

fastchat/serve/gradio_web_server.py

@@ -66,13 +66,15 @@
 The service collects user dialogue data, including both text and images, and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license.
 
 ### Acknowledgment
-We thank [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).
+We thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).
 
 <div class="sponsor-image-about">
+    <img src="https://storage.googleapis.com/public-arena-asset/skylab.png" alt="SkyLab">
     <img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
     <img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
     <img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
     <img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
+    <img src="https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png" alt="Hyperbolic">
     <img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
     <img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
 </div>
@@ -490,6 +492,9 @@ def bot_response(
             if recommended_config is not None:
                 temperature = recommended_config.get("temperature", temperature)
                 top_p = recommended_config.get("top_p", top_p)
+                max_new_tokens = recommended_config.get(
+                    "max_new_tokens", max_new_tokens
+                )
 
         stream_iter = get_api_provider_stream_iter(
             conv,
@@ -617,10 +622,10 @@ def bot_response(
     padding-bottom: 6px;
 }
 #arena_leaderboard_dataframe table {
-    font-size: 115%;
+    font-size: 110%;
 }
 #full_leaderboard_dataframe table {
-    font-size: 115%;
+    font-size: 110%;
 }
 #model_description_markdown {
     font-size: 110% !important;
@@ -644,9 +649,6 @@ def bot_response(
 #chatbot .prose {
     font-size: 105% !important;
 }
-footer {
-    display:none !important;
-}
 .sponsor-image-about img {
     margin: 0 20px;
     margin-top: 20px;
@@ -750,13 +752,15 @@ def build_about():
 
 ## Acknowledgment
 We thank [SkyPilot](https://github.com/skypilot-org/skypilot) and [Gradio](https://github.com/gradio-app/gradio) team for their system support.
-We also thank [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous sponsorship. Learn more about partnership [here](https://lmsys.org/donations/).
+We also thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous sponsorship. Learn more about partnership [here](https://lmsys.org/donations/).
 
 <div class="sponsor-image-about">
+    <img src="https://storage.googleapis.com/public-arena-asset/skylab.png" alt="SkyLab">
     <img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
     <img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
     <img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
     <img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
+    <img src="https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png" alt="Hyperbolic">
     <img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
     <img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
 </div>

fastchat/serve/gradio_web_server_multi.py

@@ -287,4 +287,5 @@ def build_demo(models, vl_models, elo_results_file, leaderboard_table_file):
         max_threads=200,
         auth=auth,
         root_path=args.gradio_root_path,
+        show_api=False,
     )

fastchat/serve/monitor/clean_chat_data.py

@@ -89,7 +89,7 @@ def clean_chat_data(log_files, action_type):
         if not isinstance(model, str):
             ct_invalid += 1
             continue
-        model = replace_model_name(model)
+        model = replace_model_name(model, row["tstamp"])
 
         try:
             lang_code = detect_language(state["messages"][state["offset"]][1])