diff --git a/src/teradata_mcp_server/server.py b/src/teradata_mcp_server/server.py index 973ab54..832e832 100644 --- a/src/teradata_mcp_server/server.py +++ b/src/teradata_mcp_server/server.py @@ -26,6 +26,8 @@ def parse_args_to_settings() -> Settings: parser.add_argument('--database_uri', type=str, required=False, help='Override DATABASE_URI connection string') parser.add_argument('--auth_mode', type=str, required=False) parser.add_argument('--auth_cache_ttl', type=int, required=False) + parser.add_argument('--logging_level', type=str, required=False) + args, _ = parser.parse_known_args() env = settings_from_env() @@ -38,7 +40,7 @@ def parse_args_to_settings() -> Settings: mcp_path=args.mcp_path if args.mcp_path is not None else env.mcp_path, auth_mode=(args.auth_mode or env.auth_mode).lower(), auth_cache_ttl=args.auth_cache_ttl if args.auth_cache_ttl is not None else env.auth_cache_ttl, - logging_level=env.logging_level, + logging_level=(args.logging_level or env.logging_level).upper(), ) diff --git a/tests/mcp_bench/.gitignore b/tests/mcp_bench/.gitignore new file mode 100644 index 0000000..a2e5a1d --- /dev/null +++ b/tests/mcp_bench/.gitignore @@ -0,0 +1,25 @@ +# Python +__pycache__/ +*.py[cod] +*.so +*.egg-info/ +.pytest_cache/ + +# Virtual environments +env/ +venv/ +.venv/ + +# Output and reports +var/ +*.log + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/tests/mcp_bench/README.md b/tests/mcp_bench/README.md index 4f9f0de..ce08c63 100644 --- a/tests/mcp_bench/README.md +++ b/tests/mcp_bench/README.md @@ -1,6 +1,6 @@ # MCP Performance Benchmark Tool -A performance testing tool for MCP (Model Context Protocol) servers. +A performance testing tool for MCP (Model Context Protocol) servers supporting multiple concurrent streams of test cases with authentication support and detailed performance reporting. ## Quick Start @@ -12,23 +12,85 @@ pip install -r requirements.txt ### 2. Start Your MCP Server -Ensure your MCP server is running. For example: +Ensure your MCP server is running in streamable http and accessible. For example: ```bash +uv run python -m teradata_mcp_server.server --mcp_transport streamable-http --mcp_port 8001 --auth_mode none # Your server should be running at http://localhost:8001/mcp/ ``` +For authenticated testing: +```bash +uv run python -m teradata_mcp_server.server --mcp_transport streamable-http --mcp_port 8001 --auth_mode Basic +``` + ### 3. Run Performance Test +**Simple Test (no auth):** +```bash +python run_perf_test.py configs/scenario_simple.json +``` + +**Authenticated Test:** ```bash -python run_perf_test.py configs/perf_test.json +export AUTH_TOKEN="ZGVtb191c2VyOmRlbW9fdXNlcg==" +python run_perf_test.py configs/scenario_simple_auth.json ``` ## Configuration +We separately define configuration files for *test cases* and *scenarios*. + +Test cases files define the list of MCP tool calls that will be issued, and scenarios the streams that will execute the cases. +A single case file may contain multiple tool calls, they will be executed sequentially in the order they are defined. +A scenario file may contain multiple stream definitions, they will be executed concurrently. Streams may be configured to loop over the test cases definitions and end after a specific duration. + ### Basic Configuration -Create a JSON configuration file with server details and test streams: +We provide multiple case and scenario files: +- `cases_mixed.json` A small sample of mixed "base" tool calls. +- `cases_tactical.json` A series of tactical queries using the `base_readQuery` tool. +- `cases_error.json` Erroneous tool calls (using wrong/missing parameters or tool names). +- `scenario_simple`: Single stream quick test (5 seconds, no loop). +- `scenario_concurrence`: Three concurrent streams running the three test cases files above in loop for 30 seconds. +- `scenario_load`: 50 concurrent streams running the cases above in loop for 5 minutes. +- `scenario_simple_auth`: Basic authentication testing with a single stream. +- `scenario_env_example`: Example configuration showing environment variable usage. + +### Creating your own scenarios + +You may create new cases and scenario JSON files to configure your own test scenarios: + +**Cases Example** + +```json +{ + "test_cases": { + "base_databaseList": [ + { + "name": "database_list_test", + "parameters": {} + } + ], + "base_readQuery": [ + { + "name": "simple_query_test", + "parameters": { + "sql": "select top 10 * from dbc.tablesv" + } + }, + { + "name": "tactical_query_test", + "parameters": { + "sql": "sel * from dbc.dbcinfo where infokey='VERSION'" + } + } + + ] + } +} +``` +**Scenario Example (no authentication)** ```json { "server": { @@ -38,7 +100,13 @@ Create a JSON configuration file with server details and test streams: "streams": [ { "stream_id": "stream_01", - "test_config": "configs/real_tools_test.json", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 30, + "loop": true + }, + { + "stream_id": "stream_02", + "test_config": "tests/mcp_bench/configs/cases_error.json", "duration": 30, "loop": true } @@ -46,56 +114,125 @@ Create a JSON configuration file with server details and test streams: } ``` -### Test Cases Configuration +**Scenario Example (with stream-level authentication)** +```json +{ + "server": { + "host": "localhost", + "port": 8001 + }, + "streams": [ + { + "stream_id": "stream_01", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 30, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_02", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 30, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN_USER2" + } + } + ] +} +``` + +## Authentication Support + +The MCP benchmark client supports Basic Authentication for testing servers that require authentication. + +### Environment Variables + +The configuration system supports environment variable expansion for secure token management: + +```bash +export AUTH_TOKEN="ZGVtb191c2VyOmRlbW9fdXNlcg==" +``` + +Configuration files can reference environment variables using: +- `$VARIABLE_NAME` syntax +- `${VARIABLE_NAME}` syntax -Define which tools/methods to test in a separate JSON file: +### Authentication Configuration + +Authentication is configured at the **stream level** (recommended for testing different users): ```json { - "test_cases": { - "tool_name": [ - { - "name": "test_name", - "parameters": { - "param1": "value1" - } + "server": { + "host": "localhost", + "port": 8001 + }, + "streams": [ + { + "stream_id": "test_01", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 5, + "loop": false, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" } - ] - } + } + ] } ``` -## Available Test Configurations +### Generating Auth Tokens + +Use the included auth helper utility to generate Basic Auth tokens: + +```bash +# Generate a token +python tests/mcp_bench/auth_helper.py encode demo_user demo_user + +# Output: +# Basic Auth Token: ZGVtb191c2VyOmRlbW9fdXNlcg== +# Authorization Header: Authorization: Basic ZGVtb191c2VyOmRlbW9fdXNlcg== + +# Decode a token (for verification) +python tests/mcp_bench/auth_helper.py decode ZGVtb191c2VyOmRlbW9fdXNlcg== + +# Output: +# Username: demo_user +# Password: demo_user +``` -- `configs/perf_test.json` - 3 concurrent streams, 30 seconds each -- `configs/minimal_test.json` - Single stream, 5 seconds (quick test) -- `configs/load_test.json` - Heavy load test with multiple streams -- `configs/real_tools_test.json` - Tests with actual MCP tools ## Example Commands -### Quick Test (5 seconds) +### Quick Test (5 seconds, no auth) ```bash -python run_perf_test.py configs/minimal_test.json +python tests/mcp_bench/run_perf_test.py tests/mcp_bench/configs/scenario_simple.json ``` -### Standard Performance Test (30 seconds) +### Authenticated Test ```bash -python run_perf_test.py configs/perf_test.json +export AUTH_TOKEN="ZGVtb191c2VyOmRlbW9fdXNlcg==" +python tests/mcp_bench/run_perf_test.py tests/mcp_bench/configs/scenario_simple_auth.json ``` -### Verbose Output (see request/response details) +### Load Test with Authentication (50 streams, 5 minutes) ```bash -python run_perf_test.py configs/minimal_test.json --verbose +export AUTH_TOKEN="ZGVtb191c2VyOmRlbW9fdXNlcg==" +python tests/mcp_bench/run_perf_test.py tests/mcp_bench/configs/scenario_load.json ``` -### Custom Server +### Verbose Output + +This enables you to see the request/response details: + ```bash -# Edit config to point to your server: -# "host": "your-server.com", "port": 8080 -python run_perf_test.py your_config.json +python tests/mcp_bench/run_perf_test.py tests/mcp_bench/configs/scenario_simple.json --verbose ``` + ## Output The tool provides: @@ -134,55 +271,71 @@ OVERALL: 📊 Detailed report saved to: var/mcp-bench/reports/perf_report_20250924_174226.json ``` -## Architecture +### Multi-User Testing -- `run_perf_test.py` - Main test runner -- `mcp_streamable_client.py` - MCP client implementation -- `configs/` - Test configuration files +You can test concurrent streams with different users by setting multiple environment variables: -## Creating Custom Tests - -1. Create a test cases file with your tools: -```json -{ - "test_cases": { - "your_tool": [ - { - "name": "your_test", - "parameters": {} - } - ] - } -} +```bash +export AUTH_TOKEN="ZGVtb191c2VyOmRlbW9fdXNlcg==" # demo_user:demo_user +export AUTH_TOKEN_USER2="YWRtaW46YWRtaW4=" # admin:admin +export AUTH_TOKEN_USER3="Z3Vlc3Q6Z3Vlc3Q=" # guest:guest ``` -2. Create a main config pointing to your test: +Then configure different streams to use different tokens: + ```json { - "server": { - "host": "localhost", - "port": 8001 - }, "streams": [ { - "stream_id": "test", - "test_config": "path/to/your/test.json", - "duration": 10, - "loop": true + "stream_id": "user1_stream", + "auth": { "Authorization": "Basic $AUTH_TOKEN" } + }, + { + "stream_id": "user2_stream", + "auth": { "Authorization": "Basic $AUTH_TOKEN_USER2" } + }, + { + "stream_id": "user3_stream", + "auth": { "Authorization": "Basic $AUTH_TOKEN_USER3" } } ] } ``` -3. Run the test: -```bash -python run_perf_test.py your_config.json +## Claude Desktop Integration + +This authentication format matches Claude Desktop MCP configurations: + +```json +{ + "mcpServers": { + "teradata-mcp-server": { + "command": "mcp-remote", + "args": [ + "http://localhost:8001/mcp/", + "--header", + "Authorization: Basic ${AUTH_TOKEN}" + ], + "env": { + "AUTH_TOKEN": "ZGVtb191c2VyOmRlbW9fdXNlcg==" + } + } + } +} ``` -## Notes +## Security Notes + +- Basic Auth tokens are Base64 encoded, not encrypted - use HTTPS in production +- Store auth tokens securely and avoid committing them to version control +- Use environment variables for sensitive authentication data +- The demo token `ZGVtb191c2VyOmRlbW9fdXNlcg==` encodes `demo_user:demo_user` -- The tool properly handles MCP session initialization -- Supports concurrent test streams -- Automatically discovers available tools on the server -- Measures response time and throughput -- Provides 100% success rate tracking \ No newline at end of file +## Architecture + +- `run_perf_test.py` - Main test runner with environment variable expansion +- `mcp_streamable_client.py` - MCP client implementation with auth support +- `auth_helper.py` - Authentication token encoding/decoding utility +- `configs/` - Test configuration files + - `scenario_*.json` - Stream configurations + - `cases_*.json` - Test case definitions \ No newline at end of file diff --git a/tests/mcp_bench/auth_helper.py b/tests/mcp_bench/auth_helper.py new file mode 100644 index 0000000..e1ea0b4 --- /dev/null +++ b/tests/mcp_bench/auth_helper.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Helper utility to generate Basic Auth tokens for MCP testing +""" + +import argparse +import base64 +import sys + + +def create_basic_auth_token(username: str, password: str) -> str: + """Create a Base64 encoded Basic auth token from username and password.""" + credentials = f"{username}:{password}" + encoded = base64.b64encode(credentials.encode('utf-8')).decode('ascii') + return encoded + + +def decode_basic_auth_token(token: str) -> tuple[str, str]: + """Decode a Base64 Basic auth token to username and password.""" + try: + decoded = base64.b64decode(token).decode('utf-8') + username, password = decoded.split(':', 1) + return username, password + except Exception as e: + raise ValueError(f"Invalid token format: {e}") + + +def main(): + parser = argparse.ArgumentParser(description="MCP Basic Auth Token Helper") + subparsers = parser.add_subparsers(dest='command', help='Available commands') + + # Encode command + encode_parser = subparsers.add_parser('encode', help='Encode username:password to Base64') + encode_parser.add_argument('username', help='Username') + encode_parser.add_argument('password', help='Password') + + # Decode command + decode_parser = subparsers.add_parser('decode', help='Decode Base64 token to username:password') + decode_parser.add_argument('token', help='Base64 encoded token') + + args = parser.parse_args() + + if args.command == 'encode': + token = create_basic_auth_token(args.username, args.password) + print(f"Basic Auth Token: {token}") + print(f"Authorization Header: Authorization: Basic {token}") + print(f"Config JSON:") + print(f' "auth": {{') + print(f' "Authorization": "Basic {token}"') + print(f' }}') + + elif args.command == 'decode': + try: + username, password = decode_basic_auth_token(args.token) + print(f"Username: {username}") + print(f"Password: {password}") + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + else: + parser.print_help() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/mcp_bench/configs/error_test.json b/tests/mcp_bench/configs/cases_error.json similarity index 100% rename from tests/mcp_bench/configs/error_test.json rename to tests/mcp_bench/configs/cases_error.json diff --git a/tests/mcp_bench/configs/tool_tests.json b/tests/mcp_bench/configs/cases_mixed.json similarity index 67% rename from tests/mcp_bench/configs/tool_tests.json rename to tests/mcp_bench/configs/cases_mixed.json index cc64a50..1a5e4b2 100644 --- a/tests/mcp_bench/configs/tool_tests.json +++ b/tests/mcp_bench/configs/cases_mixed.json @@ -10,9 +10,16 @@ { "name": "simple_query_test", "parameters": { - "sql": "select 1" + "sql": "select top 10 * from dbc.tablesv" + } + }, + { + "name": "tactical_query_test", + "parameters": { + "sql": "sel * from dbc.dbcinfo where infokey='VERSION'" } } + ], "base_tableDDL": [ { diff --git a/tests/mcp_bench/configs/cases_tactical.json b/tests/mcp_bench/configs/cases_tactical.json new file mode 100644 index 0000000..b9b93c2 --- /dev/null +++ b/tests/mcp_bench/configs/cases_tactical.json @@ -0,0 +1,66 @@ +{ + "test_cases": { + "base_readQuery": [ + { + "name": "tactical_query_test_01", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_02", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_03", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_04", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_05", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_06", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_07", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_08", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_09", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + }, + { + "name": "tactical_query_test_10", + "parameters": { + "sql": "sel day_of_week from Sys_Calendar.Calendar where calendar_date=current_date+random(-1000,1000);" + } + } + ] + } +} \ No newline at end of file diff --git a/tests/mcp_bench/configs/error_test_config.json b/tests/mcp_bench/configs/error_test_config.json deleted file mode 100644 index c0ad733..0000000 --- a/tests/mcp_bench/configs/error_test_config.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "server": { - "host": "localhost", - "port": 8001 - }, - "streams": [ - { - "stream_id": "error_test_01", - "test_config": "tests/mcp_bench/configs/error_test.json", - "duration": 5, - "loop": false - } - ] -} \ No newline at end of file diff --git a/tests/mcp_bench/configs/load_test.json b/tests/mcp_bench/configs/load_test.json deleted file mode 100644 index ec783fa..0000000 --- a/tests/mcp_bench/configs/load_test.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "server": { - "host": "localhost", - "port": 8001 - }, - "streams": [ - { - "stream_id": "stream_01", - "test_config": "tests/mcp_bench/configs/protocol_test.json", - "duration": 30, - "loop": true - }, - { - "stream_id": "stream_02", - "test_config": "tests/mcp_bench/configs/protocol_test.json", - "duration": 30, - "loop": true - }, - { - "stream_id": "stream_03", - "test_config": "tests/mcp_bench/configs/protocol_test.json", - "duration": 30, - "loop": true - } - ] -} \ No newline at end of file diff --git a/tests/mcp_bench/configs/perf_test.json b/tests/mcp_bench/configs/scenario_concurrence.json similarity index 63% rename from tests/mcp_bench/configs/perf_test.json rename to tests/mcp_bench/configs/scenario_concurrence.json index 7f41ee0..71e7f93 100644 --- a/tests/mcp_bench/configs/perf_test.json +++ b/tests/mcp_bench/configs/scenario_concurrence.json @@ -6,19 +6,19 @@ "streams": [ { "stream_id": "stream_01", - "test_config": "tests/mcp_bench/configs/tool_tests.json", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", "duration": 30, "loop": true }, { "stream_id": "stream_02", - "test_config": "tests/mcp_bench/configs/tool_tests.json", + "test_config": "tests/mcp_bench/configs/cases_error.json", "duration": 30, "loop": true }, { "stream_id": "stream_03", - "test_config": "tests/mcp_bench/configs/tool_tests.json", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", "duration": 30, "loop": true } diff --git a/tests/mcp_bench/configs/scenario_env_example.json b/tests/mcp_bench/configs/scenario_env_example.json new file mode 100644 index 0000000..f14a8f1 --- /dev/null +++ b/tests/mcp_bench/configs/scenario_env_example.json @@ -0,0 +1,26 @@ +{ + "server": { + "host": "${MCP_HOST}", + "port": "${MCP_PORT}", + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + "streams": [ + { + "stream_id": "test_01", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": "${TEST_DURATION}", + "loop": false + }, + { + "stream_id": "test_02", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": "$TEST_DURATION", + "loop": false, + "auth": { + "Authorization": "Basic ${AUTH_TOKEN}" + } + } + ] +} \ No newline at end of file diff --git a/tests/mcp_bench/configs/scenario_load.json b/tests/mcp_bench/configs/scenario_load.json new file mode 100644 index 0000000..bf22b56 --- /dev/null +++ b/tests/mcp_bench/configs/scenario_load.json @@ -0,0 +1,224 @@ +{ + "server": { + "host": "localhost", + "port": 8001 + }, + "streams": [ + { + "stream_id": "stream_01", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_02", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_03", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_04", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_05", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_06", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_07", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_08", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_09", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_10", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_11", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_12", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_01", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_02", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_03", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_04", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_05", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_06", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_07", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_08", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_09", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_10", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_11", + "test_config": "tests/mcp_bench/configs/cases_error.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + }, + { + "stream_id": "stream_12", + "test_config": "tests/mcp_bench/configs/cases_tactical.json", + "duration": 300, + "loop": true, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + } + ] +} \ No newline at end of file diff --git a/tests/mcp_bench/configs/minimal_test.json b/tests/mcp_bench/configs/scenario_simple.json similarity index 71% rename from tests/mcp_bench/configs/minimal_test.json rename to tests/mcp_bench/configs/scenario_simple.json index fca9fd8..bbc93bd 100644 --- a/tests/mcp_bench/configs/minimal_test.json +++ b/tests/mcp_bench/configs/scenario_simple.json @@ -6,7 +6,7 @@ "streams": [ { "stream_id": "test_01", - "test_config": "tests/mcp_bench/configs/tool_tests.json", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", "duration": 5, "loop": false } diff --git a/tests/mcp_bench/configs/scenario_simple_auth.json b/tests/mcp_bench/configs/scenario_simple_auth.json new file mode 100644 index 0000000..08cf671 --- /dev/null +++ b/tests/mcp_bench/configs/scenario_simple_auth.json @@ -0,0 +1,17 @@ +{ + "server": { + "host": "localhost", + "port": 8001 + }, + "streams": [ + { + "stream_id": "test_01", + "test_config": "tests/mcp_bench/configs/cases_mixed.json", + "duration": 5, + "loop": false, + "auth": { + "Authorization": "Basic $AUTH_TOKEN" + } + } + ] +} \ No newline at end of file diff --git a/tests/mcp_bench/mcp_streamable_client.py b/tests/mcp_bench/mcp_streamable_client.py index b0354b0..e892d46 100644 --- a/tests/mcp_bench/mcp_streamable_client.py +++ b/tests/mcp_bench/mcp_streamable_client.py @@ -4,6 +4,7 @@ """ import asyncio +import base64 import json import logging import time @@ -16,6 +17,7 @@ from mcp.client.streamable_http import streamablehttp_client + @dataclass class ClientMetrics: """Metrics collected for each client stream.""" @@ -67,12 +69,12 @@ def __init__( logger: Optional[logging.Logger] = None ): self.stream_id = stream_id - # Ensure we have the correct URL for MCP SDK + # Ensure we have the correct URL for MCP SDK with trailing slash if not server_url.endswith('/'): server_url += '/' if not server_url.endswith('mcp/'): server_url += 'mcp/' - self.server_url = server_url.rstrip('/') + self.server_url = server_url self.test_config_path = Path(test_config_path) self.duration_seconds = duration_seconds self.loop_tests = loop_tests @@ -262,6 +264,11 @@ async def run(self): self.logger.info(f"Connecting to {self.server_url}") + # Add small random delay to avoid race conditions with multiple clients + import random + delay = random.uniform(0, 0.5) + await asyncio.sleep(delay) + # Use MCP SDK streamablehttp_client - much simpler! async with streamablehttp_client(self.server_url, headers=self.auth) as streams: read_stream, write_stream, get_session_id_callback = streams @@ -271,9 +278,19 @@ async def run(self): write_stream, message_handler=self.message_handler ) as session: - # Initialize session - SDK handles all the protocol details - await session.initialize() - self.logger.info(f"Session initialized successfully") + # Initialize session with retry logic + max_retries = 3 + for attempt in range(max_retries): + try: + await session.initialize() + self.logger.info(f"Session initialized successfully") + break + except Exception as e: + if attempt < max_retries - 1: + self.logger.warning(f"Initialization attempt {attempt + 1} failed: {e}, retrying...") + await asyncio.sleep(1) + else: + raise # List available tools tools = await self.list_tools(session) diff --git a/tests/mcp_bench/run_perf_test.py b/tests/mcp_bench/run_perf_test.py index 428e154..a79b73d 100644 --- a/tests/mcp_bench/run_perf_test.py +++ b/tests/mcp_bench/run_perf_test.py @@ -4,6 +4,8 @@ import asyncio import json import logging +import os +import re import sys import time from pathlib import Path @@ -12,9 +14,31 @@ from mcp_streamable_client import MCPStreamableClient +def expand_env_vars(obj): + """Recursively expand environment variables in strings within a data structure.""" + if isinstance(obj, str): + # Support both $VAR and ${VAR} syntax + def replace_env_var(match): + var_name = match.group(1) or match.group(2) + return os.environ.get(var_name, match.group(0)) # Return original if not found + + # Pattern matches $VAR or ${VAR} + pattern = r'\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)' + return re.sub(pattern, replace_env_var, obj) + elif isinstance(obj, dict): + return {key: expand_env_vars(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [expand_env_vars(item) for item in obj] + else: + return obj + + def load_config(config_file: str) -> Dict: with open(config_file, 'r') as f: - return json.load(f) + config = json.load(f) + + # Expand environment variables in the configuration + return expand_env_vars(config) async def run_test(config_file: str, verbose: bool = False): @@ -30,6 +54,9 @@ async def run_test(config_file: str, verbose: bool = False): server = config['server'] server_url = f"http://{server['host']}:{server['port']}" + # Get server-level auth configuration + server_auth = server.get('auth') + print(f"\n{'='*60}") print(f"MCP PERFORMANCE TEST") print(f"Server: {server_url}") @@ -41,13 +68,16 @@ async def run_test(config_file: str, verbose: bool = False): # Create and run clients clients = [] for stream_config in config['streams']: + # Use stream-level auth if available, otherwise use server-level auth + auth_config = stream_config.get('auth', server_auth) + client = MCPStreamableClient( stream_id=stream_config.get('stream_id', 'test'), server_url=server_url, test_config_path=stream_config['test_config'], duration_seconds=stream_config.get('duration', 10), loop_tests=stream_config.get('loop', False), - auth=stream_config.get('auth') + auth=auth_config ) clients.append(client)