Merge remote-tracking branch 'origin/main' into bugfix/wasm-group-by

# Conflicts:
#	Cargo.lock
#	crates/polars-row/src/encode.rs
#	crates/polars-row/src/fixed.rs
#	crates/polars-row/src/row.rs
#	crates/polars-row/src/variable.rs

.github/CODEOWNERS

@@ -4,6 +4,7 @@
 /crates/polars-sql/  @ritchie46 @orlp @c-peters @alexander-beedie
 /crates/polars-parquet/  @ritchie46 @orlp @c-peters @coastalwhite
 /crates/polars-time/  @ritchie46 @orlp @c-peters @MarcoGorelli
+/crates/polars-python/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa
+/crates/polars-python/src/lazyframe/visit.rs  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
+/crates/polars-python/src/lazyframe/visitor/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
 /py-polars/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa
-/py-polars/src/lazyframe/visit.rs  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
-/py-polars/src/lazyframe/visitor/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-

.github/dependabot.yml

@@ -11,6 +11,10 @@ updates:
     commit-message:
       prefix: ci
     labels: ['skip changelog']
+    groups:
+      ci:
+        patterns:
+          - '*'
 
   # Rust Polars
   - package-ecosystem: cargo
@@ -21,9 +25,13 @@ updates:
       - dependency-name: '*'
         update-types: ['version-update:semver-patch']
     commit-message:
-      prefix: build(rust)
+      prefix: build
       prefix-development: chore(rust)
     labels: ['skip changelog']
+    groups:
+      rust:
+        patterns:
+          - '*'
 
   # Python Polars
   - package-ecosystem: pip
@@ -36,18 +44,10 @@ updates:
     commit-message:
       prefix: chore(python)
     labels: ['skip changelog']
-
-  - package-ecosystem: cargo
-    directory: py-polars
-    schedule:
-      interval: monthly
-    ignore:
-      - dependency-name: '*'
-        update-types: ['version-update:semver-patch']
-    commit-message:
-      prefix: build(python)
-      prefix-development: chore(python)
-    labels: ['skip changelog']
+    groups:
+      python:
+        patterns:
+          - '*'
 
   # Documentation
   - package-ecosystem: pip
@@ -60,3 +60,7 @@ updates:
     commit-message:
       prefix: chore(python)
     labels: ['skip changelog']
+    groups:
+      documentation:
+        patterns:
+          - '*'

.github/workflows/benchmark-remote.yml

@@ -2,19 +2,24 @@ name: Remote Benchmark
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - crates/**
   pull_request:
     types: [ labeled ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ github.event.label.name == 'needs-bench' }}
 
 env:
   SCALE_FACTOR: '10.0'
 
 jobs:
   main:
-    if: ${{ github.event.label.name == 'needs-bench' }}
+    if: ${{ github.ref == 'refs/heads/main' || github.event.label.name == 'needs-bench' }}
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v4
@@ -62,4 +67,11 @@ jobs:
       - name: Run benchmark
         working-directory: polars-benchmark
         run: |
-          make run-polars-no-env
+          "$HOME/py-polars-cache/run-benchmarks.sh" | tee ../py-polars/benchmark-results
+      
+      - name: Cache the Polars build
+        if: ${{ github.ref == 'refs/heads/main' }}
+        working-directory: py-polars
+        run: |
+          "$HOME/py-polars-cache/save_benchmark_data.py" "$PWD/polars" < ./benchmark-results
+          "$HOME/py-polars-cache/cache-build.sh" "$PWD/polars"

.github/workflows/benchmark.yml

@@ -82,3 +82,9 @@ jobs:
       - name: Run non-benchmark tests
         working-directory: py-polars
         run: pytest -m 'not benchmark and not debug' -n auto --dist loadgroup
+
+      - name: Run non-benchmark tests on new streaming engine
+        working-directory: py-polars
+        env:
+          POLARS_AUTO_NEW_STREAMING: 1
+        run: pytest -n auto --dist loadgroup -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"

.github/workflows/lint-global.yml

@@ -15,4 +15,4 @@ jobs:
       - name: Lint Markdown and TOML
         uses: dprint/check@v2.2
       - name: Spell Check with Typos
-        uses: crate-ci/typos@v1.24.2
+        uses: crate-ci/typos@v1.27.2

.github/workflows/lint-python.yml

@@ -39,7 +39,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.9', '3.12']
 
     steps:
       - uses: actions/checkout@v4
@@ -63,4 +63,4 @@ jobs:
       # Allow untyped calls for older Python versions
       - name: Run mypy
         working-directory: py-polars
-        run: mypy ${{ (matrix.python-version == '3.8') && '--allow-untyped-calls' || '' }}
+        run: mypy ${{ (matrix.python-version == '3.9') && '--allow-untyped-calls' || '' }}

.github/workflows/release-python.yml

@@ -18,7 +18,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  PYTHON_VERSION: '3.8'
+  PYTHON_VERSION: '3.9'
   CARGO_INCREMENTAL: 0
   CARGO_NET_RETRY: 10
   RUSTUP_MAX_RETRIES: 10
@@ -137,7 +137,6 @@ jobs:
         if: matrix.architecture == 'x86-64'
         env:
           IS_LTS_CPU: ${{ matrix.package == 'polars-lts-cpu' }}
-          IS_MACOS: ${{ matrix.os == 'macos-13' }}
         # IMPORTANT: All features enabled here should also be included in py-polars/polars/_cpu_check.py
         run: |
           if [[ "$IS_LTS_CPU" = true ]]; then
@@ -175,7 +174,7 @@ jobs:
       - name: Set variables in CPU check module - LTS_CPU
         if: matrix.package == 'polars-lts-cpu'
         run: |
-          sed $SED_INPLACE 's/^_LTS_CPU = False$/_LTS_CPU = True/g' $CPU_CHECK_MODULE
+          sed $SED_INPLACE 's/^_POLARS_LTS_CPU = False$/_POLARS_LTS_CPU = True/g' $CPU_CHECK_MODULE
 
       - name: Set Rust target for aarch64
         if: matrix.architecture == 'aarch64'
@@ -195,7 +194,7 @@ jobs:
           command: build
           target: ${{ steps.target.outputs.target }}
           args: >
-            --release
+            --profile dist-release
             --manifest-path py-polars/Cargo.toml
             --out dist
           manylinux: ${{ matrix.architecture == 'aarch64' && '2_24' || 'auto' }}

.github/workflows/test-coverage.yml

@@ -29,6 +29,8 @@ env:
   CARGO_LLVM_COV: 1
   CARGO_LLVM_COV_SHOW_ENV: 1
   CARGO_LLVM_COV_TARGET_DIR: ${{ github.workspace }}/target
+  # We use the stable ABI, silences error from PyO3 that the system Python is too new.
+  PYO3_USE_ABI3_FORWARD_COMPATIBILITY: 1
 
 jobs:
   coverage-rust:
@@ -94,9 +96,13 @@ jobs:
         with:
           python-version: '3.12'
 
-      - name: Create virtual environment
+      - name: Install uv
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Create virtual environment
+        run: |
           uv venv
           echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
           echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
@@ -163,7 +169,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-        # Needed to fetch the Codecov config file
+      # Needed to fetch the Codecov config file
       - uses: actions/checkout@v4
 
       - name: Download coverage reports

.github/workflows/test-python.yml

@@ -39,7 +39,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.11', '3.12']
+        python-version: ['3.9', '3.12']
         include:
           - os: windows-latest
             python-version: '3.12'
@@ -94,6 +94,12 @@ jobs:
         if: github.ref_name != 'main'
         run: pytest -n auto --dist loadgroup -m "not release and not benchmark and not docs"
 
+      - name: Run tests with new streaming engine
+        if: github.ref_name != 'main'
+        env:
+          POLARS_AUTO_NEW_STREAMING: 1
+        run: pytest -n auto --dist loadgroup -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"
+
       - name: Run tests async reader tests
         if: github.ref_name != 'main' && matrix.os != 'windows-latest'
         env: