Convert to tool with args

ShaharNaveh · ShaharNaveh · commit ad061b2efe09 · 2025-08-22T14:43:09.000+03:00
diff --git a/Lib/argparse.py b/Lib/argparse.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 # Author: Steven J. Bethard <steven.bethard@gmail.com>.
 # New maintainer as of 29 August 2019:  Raymond Hettinger <raymond.hettinger@gmail.com>
 
diff --git a/Lib/ast.py b/Lib/ast.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 """
 The `ast` module helps Python applications to process trees of the Python
 abstract syntax grammar.  The abstract syntax itself might change with
diff --git a/Lib/decimal.py b/Lib/decimal.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 """Decimal fixed-point and floating-point arithmetic.
 
 This is an implementation of decimal floating-point arithmetic based on
diff --git a/Lib/heapq.py b/Lib/heapq.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 """Heap queue algorithm (a.k.a. priority queue).
 
 Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 # Copyright 2007 Google Inc.
 #  Licensed to PSF under a Contributor Agreement.
 
diff --git a/Lib/numbers.py b/Lib/numbers.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 # Copyright 2007 Google, Inc. All Rights Reserved.
 # Licensed to PSF under a Contributor Agreement.
 
@@ -290,18 +291,27 @@ def conjugate(self):
 
 
 class Rational(Real):
-    """.numerator and .denominator should be in lowest terms."""
+    """To Real, Rational adds numerator and denominator properties.
+
+    The numerator and denominator values should be in lowest terms,
+    with a positive denominator.
+    """
 
     __slots__ = ()
 
     @property
     @abstractmethod
     def numerator(self):
+        """The numerator of a rational number in lowest terms."""
         raise NotImplementedError
 
     @property
     @abstractmethod
     def denominator(self):
+        """The denominator of a rational number in lowest terms.
+
+        This denominator should be positive.
+        """
         raise NotImplementedError
 
     # Concrete implementation of Real's conversion to float.
diff --git a/Lib/os.py b/Lib/os.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 r"""OS routines for NT or Posix depending on what system we're on.
 
 This exports:
@@ -10,7 +11,7 @@
   - os.extsep is the extension separator (always '.')
   - os.altsep is the alternate pathname separator (None or '/')
   - os.pathsep is the component separator used in $PATH etc
-  - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n')
+  - os.linesep is the line separator in text files ('\n' or '\r\n')
   - os.defpath is the default search path for executables
   - os.devnull is the file path of the null device ('/dev/null', etc.)
 
diff --git a/Lib/pprint.py b/Lib/pprint.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 #  Author:      Fred L. Drake, Jr.
 #               fdrake@acm.org
 #
diff --git a/Lib/queue.py b/Lib/queue.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 '''A multi-producer, multi-consumer queue.'''
 
 import threading
@@ -80,9 +81,6 @@ def task_done(self):
         have been processed (meaning that a task_done() call was received
         for every item that had been put() into the queue).
 
-        shutdown(immediate=True) calls task_done() for each remaining item in
-        the queue.
-
         Raises a ValueError if called more times than there were items
         placed in the queue.
         '''
@@ -239,9 +237,11 @@ def shutdown(self, immediate=False):
         By default, gets will only raise once the queue is empty. Set
         'immediate' to True to make gets raise immediately instead.
 
-        All blocked callers of put() and get() will be unblocked. If
-        'immediate', a task is marked as done for each item remaining in
-        the queue, which may unblock callers of join().
+        All blocked callers of put() and get() will be unblocked.
+
+        If 'immediate', the queue is drained and unfinished tasks
+        is reduced by the number of drained tasks.  If unfinished tasks
+        is reduced to zero, callers of Queue.join are unblocked.
         '''
         with self.mutex:
             self.is_shutdown = True
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 # As a test suite for the os module, this is woefully inadequate, but this
 # does add tests for a few functions which have been determined to be more
 # portable than they had been thought to be.
@@ -829,9 +830,7 @@ def ns_to_sec(ns):
         # Convert a number of nanosecond (int) to a number of seconds (float).
         # Round towards infinity by adding 0.5 nanosecond to avoid rounding
         # issue, os.utime() rounds towards minus infinity.
-        # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)`
-        # return (ns * 1e-9) + 0.5e-9
-        return (ns * 1e-9)
+        return (ns * 1e-9) + 0.5e-9
 
     def test_utime_by_indexed(self):
         # pass times as floating-point seconds as the second indexed parameter
diff --git a/Lib/this.py b/Lib/this.py
@@ -1,3 +1,4 @@
+# upstream_version: v3.13.7
 s = """Gur Mra bs Clguba, ol Gvz Crgref
 
 Ornhgvshy vf orggre guna htyl.
diff --git a/tools/rpau/README.md b/tools/rpau/README.md
@@ -0,0 +1 @@
+# **R**ust**P**ython **A**utomatic **U**pdater (rpau)
diff --git a/tools/rpau/confs/Lib/argparse.toml b/tools/rpau/confs/Lib/argparse.toml
@@ -0,0 +1 @@
+path = "Lib/argparse.py"
diff --git a/tools/rpau/confs/Lib/ast.toml b/tools/rpau/confs/Lib/ast.toml
@@ -0,0 +1 @@
+path = "Lib/ast.py"
diff --git a/tools/rpau/confs/Lib/decimal.toml b/tools/rpau/confs/Lib/decimal.toml
@@ -0,0 +1 @@
+path = "Lib/decimal.py"
diff --git a/tools/rpau/confs/Lib/heapq.toml b/tools/rpau/confs/Lib/heapq.toml
@@ -0,0 +1 @@
+path = "Lib/heapq.py"
diff --git a/tools/rpau/confs/Lib/ipaddress.toml b/tools/rpau/confs/Lib/ipaddress.toml
@@ -0,0 +1 @@
+path = "Lib/ipaddress.py"
diff --git a/tools/rpau/confs/Lib/numbers.toml b/tools/rpau/confs/Lib/numbers.toml
@@ -0,0 +1 @@
+path = "Lib/numbers.py"
diff --git a/tools/rpau/confs/Lib/os.toml b/tools/rpau/confs/Lib/os.toml
@@ -0,0 +1 @@
+path = "Lib/os.py"
diff --git a/tools/rpau/confs/Lib/pprint.toml b/tools/rpau/confs/Lib/pprint.toml
@@ -0,0 +1 @@
+path = "Lib/pprint.py"
diff --git a/tools/rpau/confs/Lib/queue.toml b/tools/rpau/confs/Lib/queue.toml
@@ -0,0 +1 @@
+path = "Lib/queue.py"
diff --git a/tools/rpau/confs/Lib/test/test_os.toml b/tools/rpau/confs/Lib/test/test_os.toml
diff --git a/tools/rpau/confs/Lib/this.toml b/tools/rpau/confs/Lib/this.toml
@@ -0,0 +1 @@
+path = "Lib/this.py"
diff --git a/tools/rpau/main.py b/tools/rpau/main.py
@@ -9,7 +9,7 @@
 
 COL_OFFSET = 4
 INDENT1 = " " * COL_OFFSET
-INDENT2 = " " * COL_OFFSET * 2
+INDENT2 = INDENT1 * 2
 COMMENT = "TODO: RUSTPYTHON"
 
 ROOT_DIR = pathlib.Path(__file__).parents[2]
diff --git a/tools/rpau/pyproject.toml b/tools/rpau/pyproject.toml
@@ -0,0 +1,17 @@
+[project]
+name = "rpau"
+version = "0.1.0"
+description = "Tool for automaticly update Lib & Tests from CPython"
+readme = "README.md"
+authors = [
+    { name = "RustPython Team", email = "" }
+]
+requires-python = ">=3.12"
+dependencies = []
+
+[project.scripts]
+rpau = "rpau:main"
+
+[build-system]
+requires = ["uv_build>=0.8.11,<0.9.0"]
+build-backend = "uv_build"
diff --git a/tools/rpau/src/rpau/__init__.py b/tools/rpau/src/rpau/__init__.py
@@ -0,0 +1,75 @@
+import functools
+from concurrent.futures import ProcessPoolExecutor
+from typing import TYPE_CHECKING
+
+import tomllib
+
+from rpau.cli import build_argparse
+from rpau.logger import build_root_logger, get_logger
+from rpau.logic import run
+
+if TYPE_CHECKING:
+    import pathlib
+    import re
+    from collections.abc import Iterator
+
+
+def iter_confs(
+    conf_dir: "pathlib.Path", *, include: "re.Pattern", exclude: "re.Pattern"
+) -> "Iterator[pathlib.Path]":
+    for conf_file in conf_dir.rglob("**/*.toml"):
+        if not conf_file.is_file():
+            continue
+
+        uri = conf_file.as_uri().removeprefix("file://")
+        if not include.match(uri):
+            continue
+
+        if exclude.match(uri):
+            continue
+
+        yield conf_file
+
+
+def main() -> None:
+    parser = build_argparse()
+    args = parser.parse_args()
+
+    logger = build_root_logger(level=args.log_level)
+    logger.debug(f"{args=}")
+
+    if args.cache:
+        logger.debug(f"Ensuring {args.cache_dir} exists")
+        cache_dir = args.cache_dir
+        cache_dir.mkdir(parents=True, exist_ok=True)
+    else:
+        cache_dir = None
+
+    conf_dir = args.conf_dir
+    with ProcessPoolExecutor(args.workers) as executor:
+        for conf_file in iter_confs(
+            conf_dir, include=args.include, exclude=args.exclude
+        ):
+            try:
+                conf = tomllib.loads(conf_file.read_text())
+            except tomllib.TOMLDecodeError as err:
+                logger.warn(f"{conf_file}: {err}")
+                continue
+
+            try:
+                path = conf.pop("path")
+            except KeyError:
+                logger.warn(f"{conf_file}: has no 'path' key. skipping")
+                continue
+
+            version = conf.pop("version", args.default_version)
+            func = functools.partial(
+                run,
+                path=path,
+                conf=conf,
+                cache_dir=cache_dir,
+                version=version,
+                output_dir=args.output_dir,
+                base_upstream_url=args.base_upstream_url,
+            )
+            executor.submit(func)
diff --git a/tools/rpau/src/rpau/cli.py b/tools/rpau/src/rpau/cli.py
@@ -0,0 +1,134 @@
+import argparse
+import logging
+import os
+import pathlib
+import re
+import sys
+
+
+class CustomArgumentParser(argparse.ArgumentParser):
+    class _CustomHelpFormatter(argparse.ArgumentDefaultsHelpFormatter):
+        def _get_help_string(self, action):
+            help_msg = super()._get_help_string(action)
+            if action.dest != "help":
+                env_name = f"{self._prog}_{action.dest}".upper()
+                env_value = os.environ.get(env_name, "")
+                help_msg += f" [env: {env_name}={env_value}]"
+            return help_msg
+
+    def __init__(self, *, formatter_class=_CustomHelpFormatter, **kwargs):
+        super().__init__(formatter_class=formatter_class, **kwargs)
+
+    def _add_action(self, action):
+        action.default = os.environ.get(
+            f"{self.prog}_{action.dest}".upper(), action.default
+        )
+        return super()._add_action(action)
+
+
+def get_cache_dir(prog: str) -> pathlib.Path:
+    home = pathlib.Path.home()
+
+    if sys.platform.startswith("win"):
+        local_appdata = pathlib.Path(
+            os.getenv("LOCALAPPDATA", home / "AppData" / "Local")
+        )
+        path = local_appdata / prog / prog / "Cache"
+    elif sys.platform == "darwin":
+        path = home / "Library" / "Caches" / prog
+    else:
+        cache_home = pathlib.Path(os.getenv("XDG_CACHE_HOME", home / ".cache"))
+        path = cache_home / prog
+
+    return path
+
+
+def build_argparse() -> argparse.ArgumentParser:
+    parser = CustomArgumentParser(
+        prog="rpau", description="Automatic update code from CPython"
+    )
+
+    # Cache
+    cache_group = parser.add_argument_group("Cache options")
+
+    cache_group.add_argument(
+        "--cache",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Whether reading from or writing to the cache is allowed",
+    )
+    cache_group.add_argument(
+        "--cache-dir",
+        default=get_cache_dir(parser.prog),
+        help="Path to the cache directory",
+        metavar="PATH",
+        type=pathlib.Path,
+    )
+
+    # Output
+    output_group = parser.add_argument_group("Output option")
+
+    output_group.add_argument(
+        "-o",
+        "--output-dir",
+        default=pathlib.Path(__file__).parents[4],
+        help="Output dir",
+        metavar="PATH",
+        type=pathlib.Path,
+    )
+
+    # Filter
+    filter_group = parser.add_argument_group("Filter options")
+
+    filter_group.add_argument(
+        "-i",
+        "--include",
+        default=".*",
+        help="RE pattern used to include files and/or directories",
+        metavar="PATTERN",
+        type=re.compile,
+    )
+    filter_group.add_argument(
+        "-e",
+        "--exclude",
+        default="^$",
+        help="RE pattern used to omit files and/or directories",
+        metavar="PATTERN",
+        type=re.compile,
+    )
+
+    # Global
+    global_group = parser.add_argument_group("Global options")
+
+    global_group.add_argument(
+        "--log-level",
+        choices=logging.getLevelNamesMapping(),
+        default="WARNING",
+        help="Log level",
+    )
+
+    global_group.add_argument(
+        "-j", "--workers", default=1, help="Number of processes", type=int
+    )
+    global_group.add_argument(
+        "-c",
+        "--conf-dir",
+        default=pathlib.Path(__file__).parents[2] / "confs",
+        help="Path to conf dir",
+        metavar="PATH",
+        type=pathlib.Path,
+    )
+    global_group.add_argument(
+        "--base-upstream-url",
+        default="https://raw.githubusercontent.com/python/cpython/refs/tags",
+        help="Base upstream url of CPython",
+        metavar="URL",
+    )
+    global_group.add_argument(
+        "--default-version",
+        default="v3.13.7",
+        help="Fallback version of cpython if none specified in conf",
+        metavar="VERSION_TAG",
+    )
+
+    return parser
diff --git a/tools/rpau/src/rpau/logger.py b/tools/rpau/src/rpau/logger.py
diff --git a/tools/rpau/src/rpau/logic.py b/tools/rpau/src/rpau/logic.py
diff --git a/tools/rpau/uv.lock b/tools/rpau/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# upstream_version: v3.13.7`
`1`	`2`	`# Author: Steven J. Bethard <steven.bethard@gmail.com>.`
`2`	`3`	`# New maintainer as of 29 August 2019: Raymond Hettinger <raymond.hettinger@gmail.com>`
`3`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# upstream_version: v3.13.7`
`1`	`2`	`"""`
`2`	`3`	The `ast` module helps Python applications to process trees of the Python
`3`	`4`	`abstract syntax grammar. The abstract syntax itself might change with`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# upstream_version: v3.13.7`
`1`	`2`	`"""Decimal fixed-point and floating-point arithmetic.`
`2`	`3`
`3`	`4`	`This is an implementation of decimal floating-point arithmetic based on`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# upstream_version: v3.13.7`
`1`	`2`	`"""Heap queue algorithm (a.k.a. priority queue).`
`2`	`3`
`3`	`4`	`Heaps are arrays for which a[k] <= a[2k+1] and a[k] <= a[2k+2] for`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# upstream_version: v3.13.7`
`1`	`2`	`# Copyright 2007 Google Inc.`
`2`	`3`	`# Licensed to PSF under a Contributor Agreement.`
`3`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+# upstream_version: v3.13.7`
`1`	`2`	`# Author: Fred L. Drake, Jr.`
`2`	`3`	`# fdrake@acm.org`
`3`	`4`	`#`