Merge branch 'main' into kevinjqliu/iceberg-write

.github/CODEOWNERS

@@ -4,6 +4,7 @@
 /crates/polars-sql/  @ritchie46 @orlp @c-peters @alexander-beedie
 /crates/polars-parquet/  @ritchie46 @orlp @c-peters @coastalwhite
 /crates/polars-time/  @ritchie46 @orlp @c-peters @MarcoGorelli
+/crates/polars-python/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa
+/crates/polars-python/src/lazyframe/visit.rs  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
+/crates/polars-python/src/lazyframe/visitor/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
 /py-polars/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa
-/py-polars/src/lazyframe/visit.rs  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
-/py-polars/src/lazyframe/visitor/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-

.github/CODE_OF_CONDUCT.md

@@ -2,17 +2,15 @@
 
 ## Our Pledge
 
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers
+pledge to make participation in our project and our community a harassment-free experience for
+everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity
+and expression, level of experience, education, socio-economic status, nationality, personal
 appearance, race, religion, or sexual identity and orientation.
 
 ## Our Standards
 
-Examples of behavior that contributes to creating a positive environment
-include:
+Examples of behavior that contributes to creating a positive environment include:
 
 - Using welcoming and inclusive language
 - Being respectful of differing viewpoints and experiences
@@ -22,53 +20,47 @@ include:
 
 Examples of unacceptable behavior by participants include:
 
-- The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
 - Trolling, insulting/derogatory comments, and personal or political attacks
 - Public or private harassment
-- Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-- Other conduct which could reasonably be considered inappropriate in a
-  professional setting
+- Publishing others' private information, such as a physical or electronic address, without explicit
+  permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
 
 ## Our Responsibilities
 
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are
+expected to take appropriate and fair corrective action in response to any instances of unacceptable
+behavior.
 
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits,
+code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or
+to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate,
 threatening, offensive, or harmful.
 
 ## Scope
 
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
+This Code of Conduct applies within all project spaces, and it also applies when an individual is
+representing the project or its community in public spaces. Examples of representing a project or
+community include using an official project e-mail address, posting via an official social media
+account, or acting as an appointed representative at an online or offline event. Representation of a
+project may be further defined and clarified by project maintainers.
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at ritchie46@gmail.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting
+the project team at ritchie46@gmail.com. All complaints will be reviewed and investigated and will
+result in a response that is deemed necessary and appropriate to the circumstances. The project team
+is obligated to maintain confidentiality with regard to the reporter of an incident. Further details
+of specific enforcement policies may be posted separately.
 
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face
+temporary or permanent repercussions as determined by other members of the project's leadership.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at
+https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
 [homepage]: https://www.contributor-covenant.org
 

.github/dependabot.yml

@@ -11,6 +11,10 @@ updates:
     commit-message:
       prefix: ci
     labels: ['skip changelog']
+    groups:
+      ci:
+        patterns:
+          - '*'
 
   # Rust Polars
   - package-ecosystem: cargo
@@ -21,9 +25,13 @@ updates:
       - dependency-name: '*'
         update-types: ['version-update:semver-patch']
     commit-message:
-      prefix: build(rust)
+      prefix: build
       prefix-development: chore(rust)
     labels: ['skip changelog']
+    groups:
+      rust:
+        patterns:
+          - '*'
 
   # Python Polars
   - package-ecosystem: pip
@@ -36,18 +44,10 @@ updates:
     commit-message:
       prefix: chore(python)
     labels: ['skip changelog']
-
-  - package-ecosystem: cargo
-    directory: py-polars
-    schedule:
-      interval: monthly
-    ignore:
-      - dependency-name: '*'
-        update-types: ['version-update:semver-patch']
-    commit-message:
-      prefix: build(python)
-      prefix-development: chore(python)
-    labels: ['skip changelog']
+    groups:
+      python:
+        patterns:
+          - '*'
 
   # Documentation
   - package-ecosystem: pip
@@ -60,3 +60,7 @@ updates:
     commit-message:
       prefix: chore(python)
     labels: ['skip changelog']
+    groups:
+      documentation:
+        patterns:
+          - '*'

.github/workflows/benchmark-remote.yml

@@ -0,0 +1,80 @@
+name: Remote Benchmark
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - crates/**
+  pull_request:
+    types: [ labeled ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event.label.name == 'needs-bench' }}
+
+env:
+  SCALE_FACTOR: '10.0'
+
+jobs:
+  main:
+    if: ${{ github.ref == 'refs/heads/main' || github.event.label.name == 'needs-bench' }}
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Clone Polars-benchmark
+        run: |
+          git clone --depth=1 https://github.com/pola-rs/polars-benchmark.git
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Create virtual environment
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
+
+      - name: Install Python dependencies
+        working-directory: py-polars
+        run: |
+          # Install typing-extensions separately whilst the `--extra-index-url` in `requirements-ci.txt`
+          # doesn't have an up-to-date typing-extensions, see
+          # https://github.com/astral-sh/uv/issues/6028#issuecomment-2287232150
+          uv pip install -U typing-extensions
+          uv pip install --compile-bytecode -r requirements-dev.txt -r requirements-ci.txt --verbose --index-strategy=unsafe-best-match
+
+      - name: Install Polars-Benchmark dependencies
+        working-directory: polars-benchmark
+        run: |
+          uv pip install --compile-bytecode -r requirements-polars-only.txt
+
+      - name: Set up Rust
+        run: rustup show
+
+      - name: Install Polars release build
+        env:
+          RUSTFLAGS: -C embed-bitcode -D warnings
+        working-directory: py-polars
+        run: |
+          maturin develop --release -- -C codegen-units=8 -C lto=thin -C target-cpu=native
+
+      - name: Run benchmark
+        working-directory: polars-benchmark
+        run: |
+          "$HOME/py-polars-cache/run-benchmarks.sh" | tee ../py-polars/benchmark-results
+
+      - name: Cache the Polars build
+        if: ${{ github.ref == 'refs/heads/main' }}
+        working-directory: py-polars
+        run: |
+          "$HOME/py-polars-cache/add-data.py" "$PWD/polars" < ./benchmark-results
+          pip install seaborn
+          "$HOME/py-polars-cache/create-plots.py"
+          touch "$HOME/py-polars-cache/upload-probe"
+          "$HOME/py-polars-cache/cache-build.sh" "$PWD/polars"

.github/workflows/benchmark.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12'
+          python-version: '3.13'
 
       - name: Set up Graphviz
         uses: ts-graphviz/setup-graphviz@v2
@@ -52,8 +52,8 @@ jobs:
           # Install typing-extensions separately whilst the `--extra-index-url` in `requirements-ci.txt`
           # doesn't have an up-to-date typing-extensions, see
           # https://github.com/astral-sh/uv/issues/6028#issuecomment-2287232150
-          uv pip install -U typing-extensions 
-          uv pip install --compile-bytecode -r requirements-dev.txt -r requirements-ci.txt --verbose
+          uv pip install -U typing-extensions
+          uv pip install --compile-bytecode -r requirements-dev.txt -r requirements-ci.txt --verbose --index-strategy=unsafe-best-match
 
       - name: Set up Rust
         run: rustup show
@@ -67,11 +67,68 @@ jobs:
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.4
 
-      - name: Install Polars release build
-        env:
-          RUSTFLAGS: -C embed-bitcode -D warnings
+      - name: Build Polars release build
         working-directory: py-polars
-        run: maturin develop --release -- -C codegen-units=8 -C lto=thin -C target-cpu=native
+        run: maturin build --profile nodebug-release -- -C codegen-units=8 -C lto=thin -C target-cpu=native
+
+      - name: Install Polars release build
+        run: |
+          pip install target/wheels/polars*.whl
+
+          # This workflow builds and installs a wheel, meaning there is no polars.abi3.so under
+          # py-polars/. This causes a binary not found error if a test tries to import polars in
+          # a Python executed using `subprocess.check_output([sys.executable])`. Fix this by
+          # symlinking the binary.
+          ln -sv \
+            $(python -c "import importlib; print(importlib.util.find_spec('polars').submodule_search_locations[0] + '/polars.abi3.so')") \
+            py-polars/polars/polars.abi3.so
+
+      - name: Set wheel size
+        run: |
+          LIB_SIZE=$(ls -l target/wheels/polars*.whl | awk '{ print $5 }')
+          echo "LIB_SIZE=$LIB_SIZE" >> $GITHUB_ENV
+
+      - name: Comment wheel size
+        uses: actions/github-script@v7
+        if: github.ref_name != 'main'
+        with:
+          script: |
+              const currentSize = process.env.LIB_SIZE || 'Unknown';
+
+              // Convert to MB
+              const currentSizeMB = currentSize !== 'Unknown' ? (currentSize / 1024 / 1024).toFixed(4) : 'Unknown';
+
+              let commentBody = `The uncompressed lib size after this PR is **${currentSizeMB} MB**.`;
+
+              const { data: comments } = await github.rest.issues.listComments({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+              });
+
+              // Look for an existing comment
+              const existingComment = comments.find(comment =>
+                comment.body.includes('The previous uncompressed lib size was')
+              );
+
+              if (existingComment) {
+                // Update the existing comment
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: existingComment.id,
+                  body: commentBody,
+                });
+              } else {
+                // Create a new comment
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: commentBody,
+                });
+              }
+        continue-on-error: true
 
       - name: Run benchmark tests
         uses: CodSpeedHQ/action@v3
@@ -82,3 +139,9 @@ jobs:
       - name: Run non-benchmark tests
         working-directory: py-polars
         run: pytest -m 'not benchmark and not debug' -n auto --dist loadgroup
+
+      - name: Run non-benchmark tests on new streaming engine
+        working-directory: py-polars
+        env:
+          POLARS_AUTO_NEW_STREAMING: 1
+        run: pytest -n auto --dist loadgroup -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"

.github/workflows/docs-global.yml

@@ -9,7 +9,6 @@ on:
   repository_dispatch:
     types:
       - python-release-docs
-  # Allow manual trigger until we have properly versioned docs
   workflow_dispatch:
 
 jobs:
@@ -39,13 +38,13 @@ jobs:
 
       - uses: chartboost/ruff-action@v1
         with:
-          src: docs/
+          src: docs/source/
           version: ${{ steps.version.outputs.version }}
           args: check --no-fix
 
       - uses: chartboost/ruff-action@v1
         with:
-          src: docs/
+          src: docs/source/
           version: ${{ steps.version.outputs.version }}
           args: format --diff
 
@@ -69,7 +68,7 @@ jobs:
           echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
 
       - name: Install Python dependencies
-        run: uv pip install -r py-polars/requirements-dev.txt -r docs/requirements.txt
+        run: uv pip install -r py-polars/requirements-dev.txt -r docs/source/requirements.txt
 
       - name: Set up Rust
         run: rustup show