Merge remote-tracking branch 'origin/main' into dcreager/callable-return

* origin/main:
  [ty] Reachability constraints: minor documentation fixes (#21774)
  [ty] Fix non-determinism in `ConstraintSet.specialize_constrained` (#21744)
  [ty] Improve `@override`, `@final` and Liskov checks in cases where there are multiple reachable definitions (#21767)
  [ty] Extend `invalid-explicit-override` to also cover properties decorated with `@override` that do not override anything (#21756)
  [ty] Enable LRU collection for parsed module (#21749)
  [ty] Support typevar-specialized dynamic types in generic type aliases (#21730)
  Add token based `parenthesized_ranges` implementation (#21738)
  [ty] Default-specialization of generic type aliases (#21765)
  [ty] Suppress false positives when `dataclasses.dataclass(...)(cls)` is called imperatively (#21729)
  [syntax-error] Default type parameter followed by non-default type parameter (#21657)

Author: dcreager

crates/ruff_db/src/parsed.rs

@@ -21,7 +21,11 @@ use crate::source::source_text;
 /// reflected in the changed AST offsets.
 /// The other reason is that Ruff's AST doesn't implement `Eq` which Salsa requires
 /// for determining if a query result is unchanged.
-#[salsa::tracked(returns(ref), no_eq, heap_size=ruff_memory_usage::heap_size)]
+///
+/// The LRU capacity of 200 was picked without any empirical evidence that it's optimal,
+/// instead it's a wild guess that it should be unlikely that incremental changes involve
+/// more than 200 modules. Parsed ASTs within the same revision are never evicted by Salsa.
+#[salsa::tracked(returns(ref), no_eq, heap_size=ruff_memory_usage::heap_size, lru=200)]
 pub fn parsed_module(db: &dyn Db, file: File) -> ParsedModule {
     let _span = tracing::trace_span!("parsed_module", ?file).entered();
 
@@ -92,14 +96,9 @@ impl ParsedModule {
         self.inner.store(None);
     }
 
-    /// Returns the pointer address of this [`ParsedModule`].
-    ///
-    /// The pointer uniquely identifies the module within the current Salsa revision,
-    /// regardless of whether particular [`ParsedModuleRef`] instances are garbage collected.
-    pub fn addr(&self) -> usize {
-        // Note that the outer `Arc` in `inner` is stable across garbage collection, while the inner
-        // `Arc` within the `ArcSwap` may change.
-        Arc::as_ptr(&self.inner).addr()
+    /// Returns the file to which this module belongs.
+    pub fn file(&self) -> File {
+        self.file
     }
 }
 

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -747,6 +747,7 @@ impl SemanticSyntaxContext for Checker<'_> {
             | SemanticSyntaxErrorKind::LoadBeforeNonlocalDeclaration { .. }
             | SemanticSyntaxErrorKind::NonlocalAndGlobal(_)
             | SemanticSyntaxErrorKind::AnnotatedGlobal(_)
+            | SemanticSyntaxErrorKind::TypeParameterDefaultOrder(_)
             | SemanticSyntaxErrorKind::AnnotatedNonlocal(_) => {
                 self.semantic_errors.borrow_mut().push(error);
             }

crates/ruff_memory_usage/src/lib.rs

@@ -1,17 +1,46 @@
-use std::sync::{LazyLock, Mutex};
+use std::cell::RefCell;
 
 use get_size2::{GetSize, StandardTracker};
 use ordermap::{OrderMap, OrderSet};
 
+thread_local! {
+    pub static TRACKER: RefCell<Option<StandardTracker>>= const { RefCell::new(None) };
+}
+
+struct TrackerGuard(Option<StandardTracker>);
+
+impl Drop for TrackerGuard {
+    fn drop(&mut self) {
+        TRACKER.set(self.0.take());
+    }
+}
+
+pub fn attach_tracker<R>(tracker: StandardTracker, f: impl FnOnce() -> R) -> R {
+    let prev = TRACKER.replace(Some(tracker));
+    let _guard = TrackerGuard(prev);
+    f()
+}
+
+fn with_tracker<F, R>(f: F) -> R
+where
+    F: FnOnce(Option<&mut StandardTracker>) -> R,
+{
+    TRACKER.with(|tracker| {
+        let mut tracker = tracker.borrow_mut();
+        f(tracker.as_mut())
+    })
+}
+
 /// Returns the memory usage of the provided object, using a global tracker to avoid
 /// double-counting shared objects.
 pub fn heap_size<T: GetSize>(value: &T) -> usize {
-    static TRACKER: LazyLock<Mutex<StandardTracker>> =
-        LazyLock::new(|| Mutex::new(StandardTracker::new()));
-
-    value
-        .get_heap_size_with_tracker(&mut *TRACKER.lock().unwrap())
-        .0
+    with_tracker(|tracker| {
+        if let Some(tracker) = tracker {
+            value.get_heap_size_with_tracker(tracker).0
+        } else {
+            value.get_heap_size()
+        }
+    })
 }
 
 /// An implementation of [`GetSize::get_heap_size`] for [`OrderSet`].

crates/ruff_python_ast/src/parenthesize.rs

@@ -11,6 +11,8 @@ use crate::ExprRef;
 /// Note that without a parent the range can be inaccurate, e.g. `f(a)` we falsely return a set of
 /// parentheses around `a` even if the parentheses actually belong to `f`. That is why you should
 /// generally prefer [`parenthesized_range`].
+///
+/// Prefer [`crate::token::parentheses_iterator`] if you have access to [`crate::token::Tokens`].
 pub fn parentheses_iterator<'a>(
     expr: ExprRef<'a>,
     parent: Option<AnyNodeRef>,
@@ -57,6 +59,8 @@ pub fn parentheses_iterator<'a>(
 
 /// Returns the [`TextRange`] of a given expression including parentheses, if the expression is
 /// parenthesized; or `None`, if the expression is not parenthesized.
+///
+/// Prefer [`crate::token::parenthesized_range`] if you have access to [`crate::token::Tokens`].
 pub fn parenthesized_range(
     expr: ExprRef,
     parent: AnyNodeRef,

crates/ruff_python_ast/src/token.rs

@@ -16,8 +16,10 @@ use crate::str_prefix::{
 use crate::{AnyStringFlags, BoolOp, Operator, StringFlags, UnaryOp};
 use ruff_text_size::{Ranged, TextRange};
 
+mod parentheses;
 mod tokens;
 
+pub use parentheses::{parentheses_iterator, parenthesized_range};
 pub use tokens::{TokenAt, TokenIterWithContext, Tokens};
 
 #[derive(Clone, Copy, PartialEq, Eq)]

crates/ruff_python_ast/src/token/parentheses.rs

@@ -0,0 +1,58 @@
+use ruff_text_size::{Ranged, TextLen, TextRange};
+
+use super::{TokenKind, Tokens};
+use crate::{AnyNodeRef, ExprRef};
+
+/// Returns an iterator over the ranges of the optional parentheses surrounding an expression.
+///
+/// E.g. for `((f()))` with `f()` as expression, the iterator returns the ranges (1, 6) and (0, 7).
+///
+/// Note that without a parent the range can be inaccurate, e.g. `f(a)` we falsely return a set of
+/// parentheses around `a` even if the parentheses actually belong to `f`. That is why you should
+/// generally prefer [`parenthesized_range`].
+pub fn parentheses_iterator<'a>(
+    expr: ExprRef<'a>,
+    parent: Option<AnyNodeRef>,
+    tokens: &'a Tokens,
+) -> impl Iterator<Item = TextRange> + 'a {
+    let after_tokens = if let Some(parent) = parent {
+        // If the parent is a node that brings its own parentheses, exclude the closing parenthesis
+        // from our search range. Otherwise, we risk matching on calls, like `func(x)`, for which
+        // the open and close parentheses are part of the `Arguments` node.
+        let exclusive_parent_end = if parent.is_arguments() {
+            parent.end() - ")".text_len()
+        } else {
+            parent.end()
+        };
+
+        tokens.in_range(TextRange::new(expr.end(), exclusive_parent_end))
+    } else {
+        tokens.after(expr.end())
+    };
+
+    let right_parens = after_tokens
+        .iter()
+        .filter(|token| !token.kind().is_trivia())
+        .take_while(move |token| token.kind() == TokenKind::Rpar);
+
+    let left_parens = tokens
+        .before(expr.start())
+        .iter()
+        .rev()
+        .filter(|token| !token.kind().is_trivia())
+        .take_while(|token| token.kind() == TokenKind::Lpar);
+
+    right_parens
+        .zip(left_parens)
+        .map(|(right, left)| TextRange::new(left.start(), right.end()))
+}
+
+/// Returns the [`TextRange`] of a given expression including parentheses, if the expression is
+/// parenthesized; or `None`, if the expression is not parenthesized.
+pub fn parenthesized_range(
+    expr: ExprRef,
+    parent: AnyNodeRef,
+    tokens: &Tokens,
+) -> Option<TextRange> {
+    parentheses_iterator(expr, Some(parent), tokens).last()
+}

crates/ruff_python_ast_integration_tests/tests/parentheses.rs

@@ -0,0 +1,199 @@
+//! Tests for [`ruff_python_ast::tokens::parentheses_iterator`] and
+//! [`ruff_python_ast::tokens::parenthesized_range`].
+
+use ruff_python_ast::{
+    self as ast, Expr,
+    token::{parentheses_iterator, parenthesized_range},
+};
+use ruff_python_parser::parse_module;
+
+#[test]
+fn test_no_parentheses() {
+    let source = "x = 2 + 2";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_single_parentheses() {
+    let source = "x = (2 + 2)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(2 + 2)");
+}
+
+#[test]
+fn test_double_parentheses() {
+    let source = "x = ((2 + 2))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "((2 + 2))");
+}
+
+#[test]
+fn test_parentheses_with_whitespace() {
+    let source = "x = (  2 + 2  )";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(  2 + 2  )");
+}
+
+#[test]
+fn test_parentheses_with_comments() {
+    let source = "x = ( # comment\n    2 + 2\n)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "( # comment\n    2 + 2\n)");
+}
+
+#[test]
+fn test_parenthesized_range_multiple() {
+    let source = "x = (((2 + 2)))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(((2 + 2)))");
+}
+
+#[test]
+fn test_parentheses_iterator_multiple() {
+    let source = "x = (((2 + 2)))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let ranges: Vec<_> =
+        parentheses_iterator(assign.value.as_ref().into(), Some(stmt.into()), tokens).collect();
+    assert_eq!(ranges.len(), 3);
+    assert_eq!(&source[ranges[0]], "(2 + 2)");
+    assert_eq!(&source[ranges[1]], "((2 + 2))");
+    assert_eq!(&source[ranges[2]], "(((2 + 2)))");
+}
+
+#[test]
+fn test_call_arguments_not_counted() {
+    let source = "f(x)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Expr(expr_stmt) = stmt else {
+        panic!("expected `Expr` statement, got {stmt:?}");
+    };
+
+    let Expr::Call(call) = expr_stmt.value.as_ref() else {
+        panic!("expected Call expression, got {:?}", expr_stmt.value);
+    };
+
+    let arg = call
+        .arguments
+        .args
+        .first()
+        .expect("call should have an argument");
+    let result = parenthesized_range(arg.into(), (&call.arguments).into(), tokens);
+    // The parentheses belong to the call, not the argument
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_call_with_parenthesized_argument() {
+    let source = "f((x))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Expr(expr_stmt) = stmt else {
+        panic!("expected Expr statement, got {stmt:?}");
+    };
+
+    let Expr::Call(call) = expr_stmt.value.as_ref() else {
+        panic!("expected `Call` expression, got {:?}", expr_stmt.value);
+    };
+
+    let arg = call
+        .arguments
+        .args
+        .first()
+        .expect("call should have an argument");
+    let result = parenthesized_range(arg.into(), (&call.arguments).into(), tokens);
+
+    let range = result.expect("should find parentheses around argument");
+    assert_eq!(&source[range], "(x)");
+}
+
+#[test]
+fn test_multiline_with_parentheses() {
+    let source = "x = (\n    2 + 2 + 2\n)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(\n    2 + 2 + 2\n)");
+}

crates/ruff_python_parser/resources/inline/err/type_parameter_default_order.py

@@ -0,0 +1,3 @@
+class C[T = int, U]: ...
+class C[T1, T2 = int, T3, T4]: ...
+type Alias[T = int, U] = ...