[GR-71679] Fix generator frame sync

PullRequest: graalpython/4127

Author: msimacek

graalpython/com.oracle.graal.python.test/src/tests/test_generators.py

@@ -2,6 +2,7 @@
 # Copyright (C) 1996-2017 Python Software Foundation
 #
 # Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+import itertools
 import sys
 import os
 import unittest
@@ -506,4 +507,130 @@ def b():
     assert gen_b.gi_yieldfrom.gi_code.co_name == 'q'
 
     gen_b.send(None)
-    assert gen_b.gi_yieldfrom is None
+    assert gen_b.gi_yieldfrom is None
+
+
+def _test_generator_frame(checks):
+    def gen():
+        a = 1
+        yield
+        b = 2
+        yield
+        c = 3
+
+    g = gen()
+    f = None
+    if checks[0]:
+        f = g.gi_frame
+        assert f
+        assert f.f_back is None
+        assert f.f_globals == globals()
+        assert f.f_locals == {}
+        assert f.f_lineno == gen.__code__.co_firstlineno
+    next(g)
+    if checks[1]:
+        if not f:
+            f = g.gi_frame
+            assert f
+        assert f.f_back is None
+        assert f.f_globals == globals()
+        assert f.f_locals == {'a': 1}
+        assert f.f_lineno == gen.__code__.co_firstlineno + 2
+        assert f is g.gi_frame
+    next(g)
+    if checks[2]:
+        if not f:
+            f = g.gi_frame
+            assert f
+        assert f.f_back is None
+        assert f.f_globals == globals()
+        assert f.f_locals == {'a': 1, 'b': 2}
+        assert f.f_lineno == gen.__code__.co_firstlineno + 4
+        assert f is g.gi_frame
+    try:
+        next(g)
+        assert False
+    except StopIteration:
+        pass
+    if f:
+        assert f.f_back is sys._getframe()
+        assert f.f_globals is globals()
+        assert f.f_locals == {'a': 1, 'b': 2, 'c': 3}
+        # TODO GR-61955
+        if sys.implementation.name != "graalpy" or not __graalpython__.is_bytecode_dsl_interpreter:
+            assert f.f_lineno == gen.__code__.co_firstlineno + 5
+    assert not g.gi_frame
+
+
+def test_generator_frame():
+    # Generte a whole matrix of possibilities of where the frame gets created vs re-synced
+    for checks in itertools.product([False, True], repeat=3):
+        _test_generator_frame(checks)
+
+
+def test_generator_frame_from_getframe():
+    def gen():
+        a = 1
+        yield sys._getframe()
+        b = 2
+
+    g = gen()
+    f = next(g)
+    assert f
+    assert f.f_back is None
+    assert f.f_globals == globals()
+    assert f.f_locals == {'a': 1}
+    assert f.f_lineno == gen.__code__.co_firstlineno + 2
+    assert f is g.gi_frame
+    try:
+        next(g)
+        assert False
+    except StopIteration:
+        pass
+    assert f.f_back is sys._getframe()
+    assert f.f_globals is globals()
+    assert f.f_locals == {'a': 1, 'b': 2}
+    # TODO GR-61955
+    if sys.implementation.name != "graalpy" or not __graalpython__.is_bytecode_dsl_interpreter:
+        assert f.f_lineno == gen.__code__.co_firstlineno + 3
+
+
+def test_generator_frame_in_running_generator():
+    def gen():
+        self = yield
+        assert self.gi_running
+        assert not self.gi_suspended
+        assert self.gi_frame.f_lineno == gen.__code__.co_firstlineno + 4
+        assert self.gi_frame.f_globals == globals()
+        assert self.gi_frame.f_locals.get('self') is self
+        assert self.gi_frame is sys._getframe()
+
+    g = gen()
+    next(g)
+    try:
+        g.send(g)
+        assert False
+    except StopIteration:
+        pass
+
+
+def test_generator_frame_in_generator_up_stack():
+    def validate(self):
+        assert self.gi_running
+        assert not self.gi_suspended
+        assert self.gi_frame.f_lineno == gen.__code__.co_firstlineno + 2
+        assert self.gi_frame.f_globals == globals()
+        assert self.gi_frame.f_locals.get('self') is self
+        assert self.gi_frame is sys._getframe(1)
+
+    def gen():
+        self = yield
+        validate(self)
+
+    g = gen()
+    next(g)
+    try:
+        g.send(g)
+        assert False
+    except StopIteration:
+        pass

graalpython/com.oracle.graal.python.test/src/tests/test_sysconfig.py

@@ -36,10 +36,18 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import sys
+import unittest
 
 
 def test_sysconfig():
     import os, sysconfig
-    os.environ["PATH"] += os.pathsep + r" : \ " + os.pathsep;
+    os.environ["PATH"] += os.pathsep + r" : \ " + os.pathsep
     sysconfig.get_config_vars()
     # must not fail
+
+
+@unittest.skipIf(sys.implementation.name != 'graalpy', "GraalPy-only test")
+def test_sysconfigdata():
+    # Maturin loads this directly, make sure the import works
+    import _sysconfigdata

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -111,6 +111,7 @@
 import com.oracle.graal.python.builtins.modules.StringModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.StructModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.SysModuleBuiltins;
+import com.oracle.graal.python.builtins.modules.SysconfigModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.ThreadModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.TimeModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.TokenizeModuleBuiltins;
@@ -566,6 +567,7 @@ private static PythonBuiltins[] initializeBuiltins(TruffleLanguage.Env env) {
                         new WeakRefModuleBuiltins(),
                         new ReferenceTypeBuiltins(),
                         new TracemallocModuleBuiltins(),
+                        new SysconfigModuleBuiltins(),
                         // contextvars
                         new ContextVarBuiltins(),
                         new ContextBuiltins(),

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysconfigModuleBuiltins.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.modules;
+
+import static com.oracle.graal.python.nodes.BuiltinNames.J__SYSCONFIG;
+
+import java.util.Collections;
+import java.util.List;
+
+import com.oracle.graal.python.builtins.CoreFunctions;
+import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
+import com.oracle.truffle.api.dsl.NodeFactory;
+
+@CoreFunctions(defineModule = J__SYSCONFIG, isEager = true)
+public final class SysconfigModuleBuiltins extends PythonBuiltins {
+    @Override
+    protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
+        return Collections.emptyList();
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/PFrame.java

@@ -163,7 +163,7 @@ public static final class Reference {
         // by a callee frame to inform the caller that it should materialize itself when it returns.
         private boolean escaped = false;
 
-        private final Reference callerInfo;
+        private Reference callerInfo;
 
         public Reference(RootNode rootNode, Reference callerInfo) {
             this.rootNode = rootNode;
@@ -174,9 +174,8 @@ public RootNode getRootNode() {
             return rootNode;
         }
 
-        public void setBackref(PFrame.Reference backref) {
-            assert pyFrame != null : "setBackref should only be called when the PFrame escaped";
-            pyFrame.setBackref(backref);
+        public void setCallerInfo(Reference callerInfo) {
+            this.callerInfo = callerInfo;
         }
 
         public void markAsEscaped() {
@@ -307,25 +306,20 @@ public Thread getThread() {
         return thread;
     }
 
-    public PFrame.Reference getBackref() {
-        return backref;
-    }
-
-    public void setBackref(PFrame.Reference backref) {
-        // GR-41914
-        // @formatter:off
-        // assert this.backref == null || this.backref == backref : "setBackref tried to set a backref different to the one that was previously attached";
-        // @formatter:on
-        this.backref = backref;
-    }
-
     public void setLine(int line) {
         if (lockLine) {
             return;
         }
         this.line = line;
     }
 
+    public void resetLine() {
+        if (lockLine) {
+            return;
+        }
+        this.line = UNINITIALIZED_LINE;
+    }
+
     public void setLineLock(int line) {
         this.line = line;
         this.lockLine = true;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/CommonGeneratorBuiltins.java

@@ -61,6 +61,7 @@
 import com.oracle.graal.python.builtins.objects.exception.PBaseException;
 import com.oracle.graal.python.builtins.objects.exception.PrepareExceptionNode;
 import com.oracle.graal.python.builtins.objects.frame.PFrame;
+import com.oracle.graal.python.builtins.objects.function.PArguments;
 import com.oracle.graal.python.builtins.objects.traceback.PTraceback;
 import com.oracle.graal.python.builtins.objects.type.slots.TpSlotIterNext.TpIterNextBuiltin;
 import com.oracle.graal.python.lib.IteratorExhausted;
@@ -103,6 +104,7 @@
 import com.oracle.truffle.api.nodes.DirectCallNode;
 import com.oracle.truffle.api.nodes.IndirectCallNode;
 import com.oracle.truffle.api.nodes.Node;
+import com.oracle.truffle.api.nodes.RootNode;
 import com.oracle.truffle.api.profiles.InlinedBranchProfile;
 import com.oracle.truffle.api.profiles.InlinedConditionProfile;
 
@@ -183,19 +185,24 @@ static Object cachedBytecodeDSL(VirtualFrame frame, Node inliningTarget, PGenera
                  * generator root.
                  */
                 MaterializedFrame generatorFrame = self.getGeneratorFrame();
-                Object[] arguments = new Object[]{generatorFrame, sendValue};
+                Object[] callArguments = new Object[]{generatorFrame, sendValue};
+                Object[] generatorArguments = generatorFrame.getArguments();
                 if (frame == null) {
                     PythonContext context = PythonContext.get(inliningTarget);
                     PythonThreadState threadState = context.getThreadState(context.getLanguage(inliningTarget));
-                    Object state = IndirectCalleeContext.enter(threadState, generatorFrame.getArguments());
+                    Object state = IndirectCalleeContext.enter(threadState, generatorArguments);
+                    PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
+                    ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
                     try {
-                        generatorResult = callNode.call(arguments);
+                        generatorResult = callNode.call(callArguments);
                     } finally {
                         IndirectCalleeContext.exit(threadState, state);
                     }
                 } else {
-                    callContext.executePrepareCall(frame, generatorFrame.getArguments(), rootNode.getCallerFlags());
-                    generatorResult = callNode.call(arguments);
+                    callContext.executePrepareCall(frame, generatorArguments, rootNode.getCallerFlags());
+                    PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
+                    ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
+                    generatorResult = callNode.call(callArguments);
                 }
             } catch (PException e) {
                 throw handleException(self, inliningTarget, errorProfile, raiseNode, e);
@@ -250,19 +257,24 @@ static Object genericBytecodeDSL(VirtualFrame frame, Node inliningTarget, PGener
                 // See the cached specialization for notes about the arguments handling
                 PRootNode rootNode = PGenerator.unwrapContinuationRoot((ContinuationRootNode) callTarget.getRootNode());
                 MaterializedFrame generatorFrame = self.getGeneratorFrame();
-                Object[] arguments = new Object[]{generatorFrame, sendValue};
+                Object[] callArguments = new Object[]{generatorFrame, sendValue};
+                Object[] generatorArguments = generatorFrame.getArguments();
                 if (frame == null) {
                     PythonContext context = PythonContext.get(inliningTarget);
                     PythonThreadState threadState = context.getThreadState(context.getLanguage(inliningTarget));
-                    Object state = IndirectCalleeContext.enter(threadState, generatorFrame.getArguments());
+                    Object state = IndirectCalleeContext.enter(threadState, generatorArguments);
+                    PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
+                    ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
                     try {
-                        generatorResult = callNode.call(callTarget, arguments);
+                        generatorResult = callNode.call(callTarget, callArguments);
                     } finally {
                         IndirectCalleeContext.exit(threadState, state);
                     }
                 } else {
-                    callContext.executePrepareCall(frame, generatorFrame.getArguments(), rootNode.getCallerFlags());
-                    generatorResult = callNode.call(callTarget, arguments);
+                    callContext.executePrepareCall(frame, generatorArguments, rootNode.getCallerFlags());
+                    PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
+                    ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
+                    generatorResult = callNode.call(callTarget, callArguments);
                 }
             } catch (PException e) {
                 throw handleException(self, inliningTarget, errorProfile, raiseNode, e);
@@ -388,9 +400,16 @@ static Object sendThrow(VirtualFrame frame, PGenerator self, Object typ, Object
                 // Instead, we throw the exception here and fake entering the generator by adding
                 // its frame to the traceback manually.
                 self.markAsFinished();
-                Node location = self.getCurrentCallTarget().getRootNode();
+                Node location;
+                RootNode rootNode = self.getCurrentCallTarget().getRootNode();
+                if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
+                    location = self.getBytecodeNode();
+                } else {
+                    location = rootNode;
+                }
                 MaterializedFrame generatorFrame = self.getGeneratorFrame();
-                PFrame pFrame = MaterializeFrameNode.materializeGeneratorFrame(location, generatorFrame, PFrame.Reference.EMPTY);
+                PFrame.Reference ref = new PFrame.Reference(rootNode, PFrame.Reference.EMPTY);
+                PFrame pFrame = MaterializeFrameNode.materializeGeneratorFrame(location, generatorFrame, self.getGlobals(), ref);
                 FrameInfo info = (FrameInfo) generatorFrame.getFrameDescriptor().getInfo();
                 pFrame.setLine(info.getFirstLineNumber());
                 Object existingTracebackObj = getTracebackNode.execute(inliningTarget, instance);