Allow newlines after function headers without docstrings (#21110)

Summary
--

This is a first step toward fixing #9745. After reviewing our open
issues and several Black issues and PRs, I personally found the function
case the most compelling, especially with very long argument lists:

```py
def func(
	self,
	arg1: int,
	arg2: bool,
	arg3: bool,
	arg4: float,
	arg5: bool,
) -> tuple[...]:
	if arg2 and arg3:
		raise ValueError
```

or many annotations:

```py
def function(
    self, data: torch.Tensor | tuple[torch.Tensor, ...], other_argument: int
) -> torch.Tensor | tuple[torch.Tensor, ...]:
    do_something(data)
    return something
```

I think docstrings help the situation substantially both because syntax
highlighting will usually give a very clear separation between the
annotations and the docstring and because we already allow a blank line
_after_ the docstring:

```py
def function(
    self, data: torch.Tensor | tuple[torch.Tensor, ...], other_argument: int
) -> torch.Tensor | tuple[torch.Tensor, ...]:
    """
	A function doing something.

	And a longer description of the things it does.
	"""

    do_something(data)
    return something
```

There are still other comments on #9745, such as [this one] with 9
upvotes, where users specifically request blank lines in all block
types, or at least including conditionals and loops. I'm sympathetic to
that case as well, even if personally I don't find an [example] like
this:

```py
if blah:

    # Do some stuff that is logically related
    data = get_data()

    # Do some different stuff that is logically related
    results = calculate_results()

    return results
```

to be much more readable than:

```py
if blah:
    # Do some stuff that is logically related
    data = get_data()

    # Do some different stuff that is logically related
    results = calculate_results()

    return results
```

I'm probably just used to the latter from the formatters I've used, but
I do prefer it. I also think that functions are the least susceptible to
the accidental introduction of a newline after refactoring described in
Micha's [comment] on #8893.

I actually considered further restricting this change to functions with
multiline headers. I don't think very short functions like:

```py
def foo():

    return 1
```

benefit nearly as much from the allowed newline, but I just went with
any function without a docstring for now. I guess a marginal case like:

```py
def foo(a_long_parameter: ALongType, b_long_parameter: BLongType) -> CLongType:

    return 1
```

might be a good argument for not restricting it.

I caused a couple of syntax errors before adding special handling for
the ellipsis-only case, so I suspect that there are some other
interesting edge cases that may need to be handled better.

Test Plan
--

Existing tests, plus a few simple new ones. As noted above, I suspect
that we may need a few more for edge cases I haven't considered.

[this one]:
#9745 (comment)
[example]:
psf/black#902 (comment)
[comment]:
#8893 (comment)

Author: ntBre

crates/ruff_python_formatter/resources/test/fixtures/ruff/newlines.py

@@ -335,3 +335,96 @@ def overload4():
     # trailing comment
 
 def overload4(a: int): ...
+
+
+# In preview, we preserve these newlines at the start of functions:
+def preserved1():
+
+    return 1
+
+def preserved2():
+
+    pass
+
+def preserved3():
+
+    def inner(): ...
+
+def preserved4():
+
+    def inner():
+        print("with a body")
+        return 1
+
+    return 2
+
+def preserved5():
+
+    ...
+    # trailing comment prevents collapsing the stub
+
+
+def preserved6():
+
+    # Comment
+
+    return 1
+
+
+def preserved7():
+
+    # comment
+    # another line
+    # and a third
+
+    return 0
+
+
+def preserved8():  # this also prevents collapsing the stub
+
+    ...
+
+
+# But we still discard these newlines:
+def removed1():
+
+    "Docstring"
+
+    return 1
+
+
+def removed2():
+
+    ...
+
+
+def removed3():
+
+    ...  # trailing same-line comment does not prevent collapsing the stub
+
+
+# And we discard empty lines after the first:
+def partially_preserved1():
+
+
+    return 1
+
+
+# We only preserve blank lines, not add new ones
+def untouched1():
+    # comment
+
+    return 0
+
+
+def untouched2():
+    # comment
+    return 0
+
+
+def untouched3():
+    # comment
+    # another line
+    # and a third
+
+    return 0

crates/ruff_python_formatter/resources/test/fixtures/ruff/range_formatting/indent.py

@@ -61,3 +61,9 @@ def test6 ():
     print("Format" )
     print(3 +  4)<RANGE_END>
     print("Format to fix indentation" )
+
+
+def test7 ():
+    <RANGE_START>print("Format" )
+    print(3 +  4)<RANGE_END>
+    print("Format to fix indentation" )

crates/ruff_python_formatter/src/preview.rs

@@ -36,3 +36,10 @@ pub(crate) const fn is_remove_parens_around_except_types_enabled(
 ) -> bool {
     context.is_preview()
 }
+
+/// Returns `true` if the
+/// [`allow_newline_after_block_open`](https://github.com/astral-sh/ruff/pull/21110) preview style
+/// is enabled.
+pub(crate) const fn is_allow_newline_after_block_open_enabled(context: &PyFormatContext) -> bool {
+    context.is_preview()
+}

crates/ruff_python_formatter/src/statement/clause.rs

@@ -8,7 +8,7 @@ use ruff_python_trivia::{SimpleToken, SimpleTokenKind, SimpleTokenizer};
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::comments::{SourceComment, leading_alternate_branch_comments, trailing_comments};
-use crate::statement::suite::{SuiteKind, contains_only_an_ellipsis};
+use crate::statement::suite::{SuiteKind, as_only_an_ellipsis};
 use crate::verbatim::write_suppressed_clause_header;
 use crate::{has_skip_comment, prelude::*};
 
@@ -449,17 +449,10 @@ impl Format<PyFormatContext<'_>> for FormatClauseBody<'_> {
             || matches!(self.kind, SuiteKind::Function | SuiteKind::Class);
 
         if should_collapse_stub
-            && contains_only_an_ellipsis(self.body, f.context().comments())
+            && let Some(ellipsis) = as_only_an_ellipsis(self.body, f.context().comments())
             && self.trailing_comments.is_empty()
         {
-            write!(
-                f,
-                [
-                    space(),
-                    self.body.format().with_options(self.kind),
-                    hard_line_break()
-                ]
-            )
+            write!(f, [space(), ellipsis.format(), hard_line_break()])
         } else {
             write!(
                 f,

crates/ruff_python_formatter/src/statement/suite.rs

@@ -13,7 +13,9 @@ use crate::comments::{
 use crate::context::{NodeLevel, TopLevelStatementPosition, WithIndentLevel, WithNodeLevel};
 use crate::other::string_literal::StringLiteralKind;
 use crate::prelude::*;
-use crate::preview::is_blank_line_before_decorated_class_in_stub_enabled;
+use crate::preview::{
+    is_allow_newline_after_block_open_enabled, is_blank_line_before_decorated_class_in_stub_enabled,
+};
 use crate::statement::stmt_expr::FormatStmtExpr;
 use crate::verbatim::{
     suppressed_node, write_suppressed_statements_starting_with_leading_comment,
@@ -169,6 +171,22 @@ impl FormatRule<Suite, PyFormatContext<'_>> for FormatSuite {
                 false,
             )
         } else {
+            // Allow an empty line after a function header in preview, if the function has no
+            // docstring and no initial comment.
+            let allow_newline_after_block_open =
+                is_allow_newline_after_block_open_enabled(f.context())
+                    && matches!(self.kind, SuiteKind::Function)
+                    && matches!(first, SuiteChildStatement::Other(_));
+
+            let start = comments
+                .leading(first)
+                .first()
+                .map_or_else(|| first.start(), Ranged::start);
+
+            if allow_newline_after_block_open && lines_before(start, f.context().source()) > 1 {
+                empty_line().fmt(f)?;
+            }
+
             first.fmt(f)?;
 
             let empty_line_after_docstring = if matches!(first, SuiteChildStatement::Docstring(_))
@@ -218,7 +236,7 @@ impl FormatRule<Suite, PyFormatContext<'_>> for FormatSuite {
                     )?;
                 } else {
                     // Preserve empty lines after a stub implementation but don't insert a new one if there isn't any present in the source.
-                    // This is useful when having multiple function overloads that should be grouped to getter by omitting new lines between them.
+                    // This is useful when having multiple function overloads that should be grouped together by omitting new lines between them.
                     let is_preceding_stub_function_without_empty_line = following
                         .is_function_def_stmt()
                         && preceding
@@ -728,17 +746,21 @@ fn stub_suite_can_omit_empty_line(preceding: &Stmt, following: &Stmt, f: &PyForm
 
 /// Returns `true` if a function or class body contains only an ellipsis with no comments.
 pub(crate) fn contains_only_an_ellipsis(body: &[Stmt], comments: &Comments) -> bool {
-    match body {
-        [Stmt::Expr(ast::StmtExpr { value, .. })] => {
-            let [node] = body else {
-                return false;
-            };
-            value.is_ellipsis_literal_expr()
-                && !comments.has_leading(node)
-                && !comments.has_trailing_own_line(node)
-        }
-        _ => false,
+    as_only_an_ellipsis(body, comments).is_some()
+}
+
+/// Returns `Some(Stmt::Ellipsis)` if a function or class body contains only an ellipsis with no
+/// comments.
+pub(crate) fn as_only_an_ellipsis<'a>(body: &'a [Stmt], comments: &Comments) -> Option<&'a Stmt> {
+    if let [node @ Stmt::Expr(ast::StmtExpr { value, .. })] = body
+        && value.is_ellipsis_literal_expr()
+        && !comments.has_leading(node)
+        && !comments.has_trailing_own_line(node)
+    {
+        return Some(node);
     }
+
+    None
 }
 
 /// Returns `true` if a [`Stmt`] is a class or function definition.