Update benchmarks to implement the PolyBench Benchmark Interface; update stable-run config based on local experiments; filter out infra files from benchmark files; install venv using system python executable, if necessary.

Author: Andrija Kolic

graalpython/com.oracle.graal.python.benchmarks/python/harness.py

@@ -209,7 +209,11 @@ def ccompile(name, code):
 
     rootdir = os.path.dirname(__file__)
     while os.path.basename(rootdir) != 'graalpython':
+        prevdir = rootdir
         rootdir = os.path.dirname(rootdir)
+        if rootdir == prevdir:
+            # Reached the root of the file system
+            raise RuntimeError("'graalpython' directory not found in path!")
 
     sys.path.append(os.path.join(
         rootdir,

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-arith-binop.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -102,6 +102,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_arith_binop_module", code)
 from c_arith_binop_module import FloatSubclass
 
@@ -127,3 +133,28 @@ def measure(num):
 
 def __benchmark__(num=5):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=2)
+
+
+def warmupIterations():
+    return 25
+
+
+def iterations():
+    return 30
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.0,
+        "upper-threshold": 0.7,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-arith-binop2.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -191,6 +191,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_arith_binop_module", code)
 from c_arith_binop_module import FloatSubclass
 
@@ -207,12 +213,37 @@ def docompute(num):
     return sum_, type(sum_)
 
 
-def measure(num):
+def measure(num, compute_num):
     sum_ = 0
     for run in range(num):
-        sum_ += docompute(100)[0]
+        sum_ += docompute(compute_num)[0]
     return sum_
 
 
-def __benchmark__(num=5):
-    return measure(num)
+def __benchmark__(num=5, compute_num=100):
+    return measure(num, compute_num)
+
+
+def run():
+    __benchmark__(num=1, compute_num=70)
+
+
+def warmupIterations():
+    return 60
+
+
+def iterations():
+    return 15
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.0,
+        "upper-threshold": 0.4,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-call-classmethod.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -125,6 +125,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_classmethod", code)
 from c_classmethod import NativeCustomType
 
@@ -143,3 +149,28 @@ def measure(num):
 
 def __benchmark__(num=1000000):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=10000)
+
+
+def warmupIterations():
+    return 50
+
+
+def iterations():
+    return 40
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.0,
+        "upper-threshold": 0.4,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-call-method-int-float.py

@@ -65,6 +65,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_min_method_module", code)
 from c_min_method_module import as_double
 
@@ -83,3 +89,28 @@ def measure(num):
 
 def __benchmark__(num=1000000):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=40000)
+
+
+def warmupIterations():
+    return 25
+
+
+def iterations():
+    return 35
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.0,
+        "upper-threshold": 0.5,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-call-method.py

@@ -141,6 +141,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_method_module", code)
 from c_method_module import NativeCustomType
 
@@ -160,3 +166,28 @@ def measure(num):
 
 def __benchmark__(num=1000000):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=3000)
+
+
+def warmupIterations():
+    return 0
+
+
+def iterations():
+    return 10
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.0,
+        "upper-threshold": 0.7,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-instantiate-large.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -119,6 +119,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_instantiation", code)
 import c_instantiation
 
@@ -138,3 +144,28 @@ def measure(num):
 
 def __benchmark__(num=1000000):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=10)
+
+
+def warmupIterations():
+    return 0
+
+
+def iterations():
+    return 70
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.1,
+        "upper-threshold": 0.2,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-issubtype-monorphic.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -124,6 +124,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_classmethod", code)
 from c_classmethod import NativeCustomType
 
@@ -142,3 +148,28 @@ def measure(num):
 
 def __benchmark__(num=1000000):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=10000)
+
+
+def warmupIterations():
+    return 10
+
+
+def iterations():
+    return 20
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.1,
+        "upper-threshold": 0.4,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]

graalpython/com.oracle.graal.python.benchmarks/python/micro/c-issubtype-polymorphic-forced-to-native.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -128,6 +128,12 @@
 """
 
 
+import sys
+import os
+# Add benchmark directory to path to allow import of harness.py
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from harness import ccompile
+
 ccompile("c_classmethod", code)
 from c_classmethod import NativeCustomType
 
@@ -147,3 +153,28 @@ def measure(num):
 
 def __benchmark__(num=1000000):
     return measure(num)
+
+
+def run():
+    __benchmark__(num=10000)
+
+
+def warmupIterations():
+    return 0
+
+
+def iterations():
+    return 30
+
+
+def summary():
+    return {
+        "name": "OutlierRemovalAverageSummary",
+        "lower-threshold": 0.1,
+        "upper-threshold": 0.4,
+    }
+
+
+def dependencies():
+    # Required to run `ccompile`
+    return ["harness.py", "tests/__init__.py"]