From 68b40f62e26dd6664786bb4dd677c7b040840507 Mon Sep 17 00:00:00 2001 From: Jacek Laskowski Date: Mon, 20 Jan 2025 08:54:14 +0100 Subject: [PATCH] Reattachable Plan Execution --- docs/client/CustomSparkConnectBlockingStub.md | 15 ++++++ ...ExecutePlanResponseReattachableIterator.md | 13 ++++++ docs/client/SparkConnectClient.md | 12 +++++ docs/reattachable-plan-execution.md | 7 +++ docs/server/ExecuteGrpcResponseSender.md | 46 +++++++++++++++++++ docs/server/ExecuteHolder.md | 19 ++++++++ docs/server/SparkConnectExecutePlanHandler.md | 27 ++++++++++- .../SparkConnectReattachExecuteHandler.md | 28 +++++++++++ mkdocs.yml | 2 + 9 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 docs/client/ExecutePlanResponseReattachableIterator.md create mode 100644 docs/reattachable-plan-execution.md create mode 100644 docs/server/ExecuteGrpcResponseSender.md create mode 100644 docs/server/SparkConnectReattachExecuteHandler.md diff --git a/docs/client/CustomSparkConnectBlockingStub.md b/docs/client/CustomSparkConnectBlockingStub.md index 9c84b03..a777605 100644 --- a/docs/client/CustomSparkConnectBlockingStub.md +++ b/docs/client/CustomSparkConnectBlockingStub.md @@ -8,3 +8,18 @@ * `SparkConnectStubState` `CustomSparkConnectBlockingStub` is created along with [SparkConnectClient](SparkConnectClient.md#bstub). + +## executePlanReattachable { #executePlanReattachable } + +```scala +executePlanReattachable( + request: ExecutePlanRequest): CloseableIterator[ExecutePlanResponse] +``` + +`executePlanReattachable`...FIXME + +--- + +`executePlanReattachable` is used when: + +* `SparkConnectClient` is requested to [execute a plan](SparkConnectClient.md#execute) (with [useReattachableExecute](Configuration.md#useReattachableExecute) enabled) diff --git a/docs/client/ExecutePlanResponseReattachableIterator.md b/docs/client/ExecutePlanResponseReattachableIterator.md new file mode 100644 index 0000000..775b91d --- /dev/null +++ b/docs/client/ExecutePlanResponseReattachableIterator.md @@ -0,0 +1,13 @@ +# ExecutePlanResponseReattachableIterator + +## Creating Instance + +`ExecutePlanResponseReattachableIterator` takes the following to be created: + +* `proto.ExecutePlanRequest` +* `ManagedChannel` +* `GrpcRetryHandler` + +`ExecutePlanResponseReattachableIterator` is created when: + +* `CustomSparkConnectBlockingStub` is requested to [executePlanReattachable](CustomSparkConnectBlockingStub.md#executePlanReattachable) diff --git a/docs/client/SparkConnectClient.md b/docs/client/SparkConnectClient.md index c206b42..edcb8a0 100644 --- a/docs/client/SparkConnectClient.md +++ b/docs/client/SparkConnectClient.md @@ -64,3 +64,15 @@ In the end, `releaseSession` requests this [CustomSparkConnectBlockingStub](#bst `releaseSession` is used when: * `SparkSession` is requested to [close](../sql/SparkSession.md#close) + +## Execute Plan { #execute } + +```scala +execute( + plan: proto.Plan): CloseableIterator[proto.ExecutePlanResponse] +``` + +`execute`...FIXME + +??? note "Not used" + `execute` does not seem used. diff --git a/docs/reattachable-plan-execution.md b/docs/reattachable-plan-execution.md new file mode 100644 index 0000000..571e04b --- /dev/null +++ b/docs/reattachable-plan-execution.md @@ -0,0 +1,7 @@ +# Reattachable Plan Execution + +**Reattachable Plan Execution** uses `executePlanReattachable` to [execute a plan](client/SparkConnectClient.md#execute). + +Reattachable Plan Execution is [enabled by default](client/Configuration.md#useReattachableExecute) + +Reattachable Plan Execution is configured using [SparkConnectClient.Builder.disableReattachableExecute](client/SparkConnectClient.Builder.md#disableReattachableExecute) and [SparkConnectClient.Builder.enableReattachableExecute](client/SparkConnectClient.Builder.md#enableReattachableExecute) methods. diff --git a/docs/server/ExecuteGrpcResponseSender.md b/docs/server/ExecuteGrpcResponseSender.md new file mode 100644 index 0000000..a9bd807 --- /dev/null +++ b/docs/server/ExecuteGrpcResponseSender.md @@ -0,0 +1,46 @@ +# ExecuteGrpcResponseSender + +## Creating Instance + +`ExecuteGrpcResponseSender` takes the following to be created: + +* [ExecuteHolder](ExecuteHolder.md) +* `StreamObserver[T]` + +`ExecuteGrpcResponseSender` is created when: + +* `SparkConnectExecutePlanHandler` is requested to [handle a ExecutePlanRequest](SparkConnectExecutePlanHandler.md#handle) +* `SparkConnectReattachExecuteHandler` is requested handle a `ReattachExecuteRequest` + +## Run { #run } + +```scala +run( + lastConsumedStreamIndex: Long): Unit +``` + +`run`...FIXME + +--- + +`run` is used when: + +* `ExecuteHolder` is requested to [runGrpcResponseSender](ExecuteHolder.md#runGrpcResponseSender) + +### Execute { #execute } + +```scala +execute( + lastConsumedStreamIndex: Long): Unit +``` + +`execute`...FIXME + +### enqueueProgressMessage { #enqueueProgressMessage } + +```scala +enqueueProgressMessage( + force: Boolean = false): Unit +``` + +`enqueueProgressMessage`...FIXME diff --git a/docs/server/ExecuteHolder.md b/docs/server/ExecuteHolder.md index f6f2dfe..1786bc1 100644 --- a/docs/server/ExecuteHolder.md +++ b/docs/server/ExecuteHolder.md @@ -10,3 +10,22 @@ `ExecuteHolder` is created when: * `SparkConnectExecutionManager` is requested to [create an ExecuteHolder](SparkConnectExecutionManager.md#createExecuteHolder) + +## runGrpcResponseSender { #runGrpcResponseSender } + +```scala +runGrpcResponseSender( + responseSender: ExecuteGrpcResponseSender[proto.ExecutePlanResponse]): Unit +runGrpcResponseSender( + responseSender: ExecuteGrpcResponseSender[proto.ExecutePlanResponse], + lastConsumedResponseId: String): Unit +``` + +`runGrpcResponseSender`...FIXME + +--- + +`runGrpcResponseSender` is used when: + +* `SparkConnectExecutePlanHandler` is requested to [handle a ExecutePlanRequest](SparkConnectExecutePlanHandler.md#handle) +* `SparkConnectReattachExecuteHandler` is requested to [handle a ReattachExecuteRequest](SparkConnectReattachExecuteHandler.md#handle) diff --git a/docs/server/SparkConnectExecutePlanHandler.md b/docs/server/SparkConnectExecutePlanHandler.md index 4012904..7a42009 100644 --- a/docs/server/SparkConnectExecutePlanHandler.md +++ b/docs/server/SparkConnectExecutePlanHandler.md @@ -1,3 +1,28 @@ # SparkConnectExecutePlanHandler -`SparkConnectExecutePlanHandler` is...FIXME +`SparkConnectExecutePlanHandler` is used by [SparkConnectService](SparkConnectService.md) to [executePlan](SparkConnectService.md#executePlan). + +## Creating Instance + +`SparkConnectExecutePlanHandler` takes the following to be created: + +* `StreamObserver[proto.ExecutePlanResponse]` + +`SparkConnectExecutePlanHandler` is created when: + +* `SparkConnectService` is requested to [executePlan](SparkConnectService.md#executePlan) + +## Handle ExecutePlanRequest { #handle } + +```scala +handle( + v: proto.ExecutePlanRequest): Unit +``` + +`handle`...FIXME + +--- + +`handle` is used when: + +* `SparkConnectService` is requested to [executePlan](SparkConnectService.md#executePlan) diff --git a/docs/server/SparkConnectReattachExecuteHandler.md b/docs/server/SparkConnectReattachExecuteHandler.md new file mode 100644 index 0000000..4926149 --- /dev/null +++ b/docs/server/SparkConnectReattachExecuteHandler.md @@ -0,0 +1,28 @@ +# SparkConnectReattachExecuteHandler + +`SparkConnectReattachExecuteHandler` is used by [SparkConnectService](SparkConnectService.md) to [reattachExecute](SparkConnectService.md#reattachExecute). + +## Creating Instance + +`SparkConnectReattachExecuteHandler` takes the following to be created: + +* `StreamObserver[proto.ExecutePlanResponse]` + +`SparkConnectReattachExecuteHandler` is created when: + +* `SparkConnectService` is requested to [reattachExecute](SparkConnectService.md#reattachExecute) + +## Handle ReattachExecuteRequest { #handle } + +```scala +handle( + v: proto.ReattachExecuteRequest): Unit +``` + +`handle`...FIXME + +--- + +`handle` is used when: + +* `SparkConnectService` is requested to [reattachExecute](SparkConnectService.md#reattachExecute) diff --git a/mkdocs.yml b/mkdocs.yml index 3c698e4..28f32c9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -128,6 +128,8 @@ extra_css: nav: - index.md + - Features: + - reattachable-plan-execution.md - Internals: - overview.md - logging.md