Add free-threading (nogil) support

Author: ZipFile

.github/workflows/publish.yml

@@ -15,7 +15,7 @@ jobs:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Build sdist
       run: pipx run build --sdist
     - uses: actions/upload-artifact@v4
@@ -31,10 +31,13 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-24.04, ubuntu-24.04-arm]
+    env:
+      CIBW_ENVIRONMENT: >-
+        CFLAGS="-g0"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Build wheels
-      uses: pypa/cibuildwheel@v2.23.3
+      uses: pypa/cibuildwheel@v3.1.4
     - uses: actions/upload-artifact@v4
       with:
         if-no-files-found: error

README.rst

@@ -7,12 +7,9 @@ Iterate large directories efficiently with python.
 About
 =====
 
-``python-getdents`` is a simple wrapper around Linux system call ``getdents64`` (see ``man getdents`` for details). `More details <http://be-n.com/spw/you-can-list-a-million-files-in-a-directory-but-not-with-ls.html>`_ on approach.
+``python-getdents`` is a simple wrapper around Linux system call ``getdents64`` (see ``man getdents`` for details).
 
-TODO
-====
-
-* Verify that implementation works on platforms other than ``x86_64``.
+Implementation is based on solution descibed in `You can list a directory containing 8 million files! But not with ls. <http://be-n.com/spw/you-can-list-a-million-files-in-a-directory-but-not-with-ls.html>`_ article by Ben Congleton.
 
 Install
 =======
@@ -45,51 +42,55 @@ Run tests
 
     ulimit -v 33554432 && py.test tests/
 
-Or
-
-.. code-block:: sh
-
-    ulimit -v 33554432 && ./setup.py test
-
 Usage
 =====
 
 .. code-block:: python
 
     from getdents import getdents
 
-    for inode, type, name in getdents('/tmp', 32768):
+    for inode, type_, name in getdents("/tmp"):
         print(name)
 
 Advanced
 --------
 
+While ``getdents`` provides a convenient wrapper with ls-like filtering, you can use ``getdents_raw`` for more control:
+
 .. code-block:: python
 
     import os
-    from getdents import *
-
-    fd = os.open('/tmp', O_GETDENTS)
-
-    for inode, type, name in getdents_raw(fd, 2**20):
-        print({
-                DT_BLK:     'blockdev',
-                DT_CHR:     'chardev ',
-                DT_DIR:     'dir     ',
-                DT_FIFO:    'pipe    ',
-                DT_LNK:     'symlink ',
-                DT_REG:     'file    ',
-                DT_SOCK:    'socket  ',
-                DT_UNKNOWN: 'unknown ',
-            }[type], {
-                True:  'd',
-                False: ' ',
-            }[inode == 0],
-            name,
-        )
+    from getdents import DT_LNK, O_GETDENTS, getdents_raw
+
+    fd = os.open("/tmp", O_GETDENTS)
+
+    for inode, type_, name in getdents_raw(fd, 2**20):
+        if type_ == DT_LNK and inode != 0:
+            print("found symlink:", name, "->", os.readlink(name, dir_fd=fd))
 
     os.close(fd)
 
+Batching
+~~~~~~~~
+
+In case you need more control over syscalls, you may call instance of ``getdents_raw`` instead.
+Each call corresponds to single ``getdents64`` syscall, returning list of hovever many entries fits in buffer size.
+Call returns ``None`` when there are no more entries to read.
+
+.. code-block:: python
+
+    it = getdents_raw(fd, 2**20)
+
+    for batch in iter(it, None):
+         for inode, type, name in batch:
+            ...
+
+Free-threading
+~~~~~~~~~~~~~~
+
+While it is not so wise idea to do an I/O from multiple threads on a single file descriptor, you can do it if you need to.
+This package supports free-threading (nogil) in Python.
+
 CLI
 ---
 

getdents/__about__.py

@@ -1,4 +1,4 @@
-__version__ = "0.4.1"
+__version__ = "1.0.0"
 
 __all__ = [
     "__version__",

getdents/__init__.py

@@ -1,6 +1,8 @@
 import os
-from typing import Iterator, Tuple
+from collections.abc import Callable
+from typing import Iterator, TypeAlias
 
+from .__about__ import __version__
 from ._getdents import (
     DT_BLK,
     DT_CHR,
@@ -15,18 +17,38 @@
     getdents_raw,
 )
 
-DirectoryEntry = Tuple[int, int, str]
+DOT_ENTRIES = (".", "..")
+DirectoryEntry: TypeAlias = tuple[
+    int,  # inode
+    int,  # type
+    str,  # name
+]
+DirectoryEntryFilterFunction: TypeAlias = Callable[[DirectoryEntry], bool]
+
+
+def ls(d: DirectoryEntry) -> bool:
+    """``ls``-like filter for raw directory entries"""
+    return not (d[0] == 0 or d[1] == DT_UNKNOWN or d[2][0] == ".")
+
 
+def ls_a(d: DirectoryEntry) -> bool:
+    """``ls -a``-like filter for raw directory entries"""
+    return not (d[0] == 0 or d[1] == DT_UNKNOWN or d[2] in DOT_ENTRIES)
 
-def getdents(path: str, buff_size: int = 32768) -> Iterator[DirectoryEntry]:
+
+def getdents(
+    path: str,
+    buff_size: int = 1048576,
+    filter_function: DirectoryEntryFilterFunction | None = ls_a,
+) -> Iterator[DirectoryEntry]:
     """Get directory entries.
 
     Wrapper around getdents_raw(), simulates ls behaviour: ignores deleted
     files, skips . and .. entries.
 
     Note:
-       Default buffer size is 32k, it's a default allocation size of glibc's
-       readdir() implementation.
+       Buffer size of glibc's readdir() is 32KiB. You probably want more, so
+       our default is set to 1MiB.
 
     Note:
        Larger buffer will result in a fewer syscalls, so for really large
@@ -37,23 +59,27 @@ def getdents(path: str, buff_size: int = 32768) -> Iterator[DirectoryEntry]:
        size for filesystem I/O.
 
     Args:
-        path (str): Location of the directory.
-        buff_size (int): Buffer size in bytes for getdents64 syscall.
+        path: Location of the directory.
+        buff_size: Buffer size in bytes for getdents64 syscall.
+        filter_function: Function to filter directory entries.
     """
 
     fd = os.open(path, O_GETDENTS)
 
     try:
-        yield from (
-            (inode, type, name)
-            for inode, type, name in getdents_raw(fd, buff_size)
-            if not (type == DT_UNKNOWN or inode == 0 or name in (".", ".."))
-        )
+        it: Iterator[DirectoryEntry] = getdents_raw(fd, buff_size)
+
+        if filter_function:
+            it = filter(filter_function, it)
+
+        yield from it
     finally:
         os.close(fd)
 
 
 __all__ = [
+    "__version__",
+    "DOT_ENTRIES",
     "DT_BLK",
     "DT_CHR",
     "DT_DIR",
@@ -67,4 +93,6 @@ def getdents(path: str, buff_size: int = 32768) -> Iterator[DirectoryEntry]:
     "DirectoryEntry",
     "getdents",
     "getdents_raw",
+    "ls",
+    "ls_a",
 ]