From 30052d7680e988804746a8fbbc1854042a6c144d Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 22 Oct 2025 16:56:30 +0300 Subject: [PATCH] Revert "Revert "Use ruff for `Expr` unparsing (#6124)"" This reverts commit 153d0eef51ea109d8a322fd351678847b2fb8fe2. --- Cargo.lock | 25 +++ Cargo.toml | 1 + ...syntax_future10.py => badsyntax_future.py} | 0 .../test_future_stmt/badsyntax_future3.py | 10 - .../test_future_stmt/badsyntax_future4.py | 10 - .../test_future_stmt/badsyntax_future5.py | 12 - .../test_future_stmt/badsyntax_future6.py | 10 - .../test_future_stmt/badsyntax_future7.py | 11 - .../test_future_stmt/badsyntax_future8.py | 10 - .../test_future_stmt/badsyntax_future9.py | 10 - ..._test1.py => import_nested_scope_twice.py} | 0 .../{future_test2.py => nested_scope.py} | 0 Lib/test/test_future_stmt/test_future.py | 206 ++++++++++++------ compiler/codegen/Cargo.toml | 3 + compiler/codegen/src/compile.rs | 17 +- compiler/codegen/src/lib.rs | 1 - 16 files changed, 179 insertions(+), 147 deletions(-) rename Lib/test/test_future_stmt/{badsyntax_future10.py => badsyntax_future.py} (100%) delete mode 100644 Lib/test/test_future_stmt/badsyntax_future3.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future4.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future5.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future6.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future7.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future8.py delete mode 100644 Lib/test/test_future_stmt/badsyntax_future9.py rename Lib/test/test_future_stmt/{future_test1.py => import_nested_scope_twice.py} (100%) rename Lib/test/test_future_stmt/{future_test2.py => nested_scope.py} (100%) diff --git a/Cargo.lock b/Cargo.lock index 53b4f96922..cc0f080fd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2282,6 +2282,29 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "ruff_python_codegen" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" +dependencies = [ + "ruff_python_ast", + "ruff_python_literal", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", +] + +[[package]] +name = "ruff_python_literal" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8" +dependencies = [ + "bitflags 2.9.4", + "itertools 0.14.0", + "ruff_python_ast", + "unic-ucd-category", +] + [[package]] name = "ruff_python_parser" version = "0.0.0" @@ -2387,7 +2410,9 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", + "ruff_python_codegen", "ruff_python_parser", + "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", diff --git a/Cargo.toml b/Cargo.toml index 3cdc471dc3..77bba61a3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,6 +159,7 @@ rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } +ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" } diff --git a/Lib/test/test_future_stmt/badsyntax_future10.py b/Lib/test/test_future_stmt/badsyntax_future.py similarity index 100% rename from Lib/test/test_future_stmt/badsyntax_future10.py rename to Lib/test/test_future_stmt/badsyntax_future.py diff --git a/Lib/test/test_future_stmt/badsyntax_future3.py b/Lib/test/test_future_stmt/badsyntax_future3.py deleted file mode 100644 index f1c8417eda..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future3.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -from __future__ import rested_snopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future4.py b/Lib/test/test_future_stmt/badsyntax_future4.py deleted file mode 100644 index b5f4c98e92..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future4.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -import __future__ -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future5.py b/Lib/test/test_future_stmt/badsyntax_future5.py deleted file mode 100644 index 8a7e5fcb70..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future5.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -import foo -from __future__ import nested_scopes - - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future6.py b/Lib/test/test_future_stmt/badsyntax_future6.py deleted file mode 100644 index 5a8b55a02c..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future6.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -"this isn't a doc string" -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future7.py b/Lib/test/test_future_stmt/badsyntax_future7.py deleted file mode 100644 index 131db2c216..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future7.py +++ /dev/null @@ -1,11 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes; import string; from __future__ import \ - nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/test_future_stmt/badsyntax_future8.py b/Lib/test/test_future_stmt/badsyntax_future8.py deleted file mode 100644 index ca45289e2e..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future8.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import * - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/badsyntax_future9.py b/Lib/test/test_future_stmt/badsyntax_future9.py deleted file mode 100644 index 916de06ab7..0000000000 --- a/Lib/test/test_future_stmt/badsyntax_future9.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes, braces - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/test_future_stmt/future_test1.py b/Lib/test/test_future_stmt/import_nested_scope_twice.py similarity index 100% rename from Lib/test/test_future_stmt/future_test1.py rename to Lib/test/test_future_stmt/import_nested_scope_twice.py diff --git a/Lib/test/test_future_stmt/future_test2.py b/Lib/test/test_future_stmt/nested_scope.py similarity index 100% rename from Lib/test/test_future_stmt/future_test2.py rename to Lib/test/test_future_stmt/nested_scope.py diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 9c30054963..e6e6201a76 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -10,6 +10,8 @@ import re import sys +TOP_LEVEL_MSG = 'from __future__ imports must occur at the beginning of the file' + rx = re.compile(r'\((\S+).py, line (\d+)') def get_error_location(msg): @@ -18,21 +20,48 @@ def get_error_location(msg): class FutureTest(unittest.TestCase): - def check_syntax_error(self, err, basename, lineno, offset=1): - self.assertIn('%s.py, line %d' % (basename, lineno), str(err)) - self.assertEqual(os.path.basename(err.filename), basename + '.py') + def check_syntax_error(self, err, basename, + *, + lineno, + message=TOP_LEVEL_MSG, offset=1): + if basename != '': + basename += '.py' + + self.assertEqual(f'{message} ({basename}, line {lineno})', str(err)) + self.assertEqual(os.path.basename(err.filename), basename) self.assertEqual(err.lineno, lineno) self.assertEqual(err.offset, offset) - def test_future1(self): - with import_helper.CleanImport('test.test_future_stmt.future_test1'): - from test.test_future_stmt import future_test1 - self.assertEqual(future_test1.result, 6) + def assertSyntaxError(self, code, + *, + lineno=1, + message=TOP_LEVEL_MSG, offset=1, + parametrize_docstring=True): + code = dedent(code.lstrip('\n')) + for add_docstring in ([False, True] if parametrize_docstring else [False]): + with self.subTest(code=code, add_docstring=add_docstring): + if add_docstring: + code = '"""Docstring"""\n' + code + lineno += 1 + with self.assertRaises(SyntaxError) as cm: + exec(code) + self.check_syntax_error(cm.exception, "", + lineno=lineno, + message=message, + offset=offset) + + def test_import_nested_scope_twice(self): + # Import the name nested_scopes twice to trigger SF bug #407394 + with import_helper.CleanImport( + 'test.test_future_stmt.import_nested_scope_twice', + ): + from test.test_future_stmt import import_nested_scope_twice + self.assertEqual(import_nested_scope_twice.result, 6) - def test_future2(self): - with import_helper.CleanImport('test.test_future_stmt.future_test2'): - from test.test_future_stmt import future_test2 - self.assertEqual(future_test2.result, 6) + def test_nested_scope(self): + with import_helper.CleanImport('test.test_future_stmt.nested_scope'): + from test.test_future_stmt import nested_scope + self.assertEqual(nested_scope.result, 6) def test_future_single_import(self): with import_helper.CleanImport( @@ -52,47 +81,87 @@ def test_future_multiple_features(self): ): from test.test_future_stmt import test_future_multiple_features - def test_badfuture3(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future3 - self.check_syntax_error(cm.exception, "badsyntax_future3", 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_unknown_future_flag(self): + code = """ + from __future__ import nested_scopes + from __future__ import rested_snopes # typo error here: nested => rested + """ + self.assertSyntaxError( + code, lineno=2, + message='future feature rested_snopes is not defined', offset=24, + ) - def test_badfuture4(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future4 - self.check_syntax_error(cm.exception, "badsyntax_future4", 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_future_import_not_on_top(self): + code = """ + import some_module + from __future__ import annotations + """ + self.assertSyntaxError(code, lineno=2) - def test_badfuture5(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future5 - self.check_syntax_error(cm.exception, "badsyntax_future5", 4) + code = """ + import __future__ + from __future__ import annotations + """ + self.assertSyntaxError(code, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_badfuture6(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future6 - self.check_syntax_error(cm.exception, "badsyntax_future6", 3) + code = """ + from __future__ import absolute_import + "spam, bar, blah" + from __future__ import print_function + """ + self.assertSyntaxError(code, lineno=3) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_future_import_with_extra_string(self): + code = """ + '''Docstring''' + "this isn't a doc string" + from __future__ import nested_scopes + """ + self.assertSyntaxError(code, lineno=3, parametrize_docstring=False) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_multiple_import_statements_on_same_line(self): + # With `\`: + code = """ + from __future__ import nested_scopes; import string; from __future__ import \ + nested_scopes + """ + self.assertSyntaxError(code, offset=54) - def test_badfuture7(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future7 - self.check_syntax_error(cm.exception, "badsyntax_future7", 3, 54) + # Without `\`: + code = """ + from __future__ import nested_scopes; import string; from __future__ import nested_scopes + """ + self.assertSyntaxError(code, offset=54) - def test_badfuture8(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future8 - self.check_syntax_error(cm.exception, "badsyntax_future8", 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_future_import_star(self): + code = """ + from __future__ import * + """ + self.assertSyntaxError(code, message='future feature * is not defined', offset=24) - def test_badfuture9(self): - with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future9 - self.check_syntax_error(cm.exception, "badsyntax_future9", 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_future_import_braces(self): + code = """ + from __future__ import braces + """ + # Congrats, you found an easter egg! + self.assertSyntaxError(code, message='not a chance', offset=24) + + code = """ + from __future__ import nested_scopes, braces + """ + self.assertSyntaxError(code, message='not a chance', offset=39) - def test_badfuture10(self): + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future10 - self.check_syntax_error(cm.exception, "badsyntax_future10", 3) + from test.test_future_stmt import badsyntax_future + self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) def test_ensure_flags_dont_clash(self): # bpo-39562: test that future flags and compiler flags doesn't clash @@ -109,26 +178,6 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) @@ -141,6 +190,26 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: future feature spam is not defined + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ @@ -198,6 +267,7 @@ def _exec_future(self, code): ) return scope + @unittest.expectedFailure # TODO: RUSTPYTHON; 'a,' != '(a,)' def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -362,6 +432,7 @@ def test_annotations(self): eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'" def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected # result explicitly. @@ -372,6 +443,7 @@ def test_fstring_debug_annotations(self): self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") + @unittest.expectedFailure # TODO: RUSTPYTHON; '1e309, 1e309j' != '(1e309, 1e309j)' def test_infinity_numbers(self): inf = "1e" + repr(sys.float_info.max_10_exp + 1) infj = f"{inf}j" @@ -384,8 +456,7 @@ def test_infinity_numbers(self): self.assertAnnotationEqual("('inf', 1e1000, 'infxxx', 1e1000j)", expected=f"('inf', {inf}, 'infxxx', {infj})") self.assertAnnotationEqual("(1e1000, (1e1000j,))", expected=f"({inf}, ({infj},))") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised def test_annotation_with_complex_target(self): with self.assertRaises(SyntaxError): exec( @@ -409,8 +480,7 @@ def bar(): self.assertEqual(foo.__code__.co_cellvars, ()) self.assertEqual(foo().__code__.co_freevars, ()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'" def test_annotations_forbidden(self): with self.assertRaises(SyntaxError): self._exec_future("test: (yield)") diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index ce7e8d74f5..e2bfec6c3c 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -13,6 +13,9 @@ rustpython-compiler-core = { workspace = true } rustpython-literal = {workspace = true } rustpython-wtf8 = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_parser = { workspace = true } +ruff_python_codegen = { workspace = true } +ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } ahash = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 5f8caebf27..132998fe83 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -14,7 +14,6 @@ use crate::{ error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, ir::{self, BlockIdx}, symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, - unparse::UnparseExpr, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -31,6 +30,7 @@ use ruff_python_ast::{ PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, }; +use ruff_source_file::LineEnding; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{ Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, @@ -147,6 +147,15 @@ enum ComprehensionType { Dict, } +fn unparse_expr(expr: &Expr) -> String { + use ruff_python_ast::str::Quote; + use ruff_python_codegen::{Generator, Indentation}; + + Generator::new(&Indentation::default(), LineEnding::default()) + .with_preferred_quote(Some(Quote::Single)) + .expr(expr) +} + fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { let mut seen_params = HashSet::new(); for param in params { @@ -3609,7 +3618,7 @@ impl Compiler { | Expr::NoneLiteral(_) ); let key_repr = if is_literal { - UnparseExpr::new(key, &self.source_file).to_string() + unparse_expr(key) } else if is_attribute { String::new() } else { @@ -4163,9 +4172,7 @@ impl Compiler { fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { - value: UnparseExpr::new(annotation, &self.source_file) - .to_string() - .into(), + value: unparse_expr(annotation).into(), }); } else { let was_in_annotation = self.in_annotation; diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index 291b57d7f6..e794407998 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -13,7 +13,6 @@ pub mod error; pub mod ir; mod string_parser; pub mod symboltable; -mod unparse; pub use compile::CompileOpts; use ruff_python_ast::Expr;